{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\hasee\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import datetime\n",
    "from sklearn.feature_selection import chi2, SelectPercentile\n",
    "from sklearn.preprocessing import OneHotEncoder, LabelEncoder\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from scipy import sparse\n",
    "import lightgbm as lgb\n",
    "import warnings\n",
    "import time\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import re\n",
    "import gc\n",
    "import pandas as pd\n",
    "import urllib.parse\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "import matplotlib.pyplot as plt\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "warnings.filterwarnings(\"ignore\")\n",
    "path = './data1/train'\n",
    "# round2 train\n",
    "train1 = pd.read_csv(path + '/operation_train_new.csv')\n",
    "train2 = pd.read_csv(path + '/transaction_train_new.csv')\n",
    "train3 = pd.read_csv(path + '/tag_train_new.csv')\n",
    "\n",
    "# round1 test\n",
    "test1_round1 = pd.read_csv(path + '/operation_round1_new.csv')\n",
    "test2_round1 = pd.read_csv(path + '/transaction_round1_new.csv')\n",
    "test1_round1['day'] = test1_round1['day'].apply(lambda x: x + 30)\n",
    "test2_round1['day'] = test2_round1['day'].apply(lambda x: x + 30)\n",
    "result_round1 = pd.read_csv(path + '/test_round1_tag.csv')\n",
    "\n",
    "operation_data = pd.concat([train1, test1_round1], axis=0, ignore_index=True)\n",
    "transaction_data = pd.concat([train2, test2_round1], axis=0, ignore_index=True)\n",
    "data = pd.concat([train3, result_round1], axis=0, ignore_index=True)\n",
    "\n",
    "\n",
    "# merchant nunique10-1000 and bal > 300\n",
    "temp = transaction_data[transaction_data.bal > 300][['UID', 'merchant']]\n",
    "temp1 = temp.copy()\n",
    "temp['merchant_nunique'] = temp['UID']\n",
    "temp = temp.groupby('merchant')['merchant_nunique'].agg('nunique').reset_index()\n",
    "merchant1 = list(temp[(temp.merchant_nunique > 10)&(temp.merchant_nunique < 1000)]['merchant'])\n",
    "temp1 = list(temp1[temp1.merchant.isin(merchant1)]['UID'].drop_duplicates())\n",
    "\n",
    "data = data[~data.UID.isin(temp1)]\n",
    "transaction_data = transaction_data[~transaction_data.UID.isin(temp1)]\n",
    "operation_data = operation_data[~operation_data.UID.isin(temp1)]\n",
    "\n",
    "\n",
    "# round2 test\n",
    "path1 = './data1/test'\n",
    "test1 = pd.read_csv(path1 + '/test_operation_round2.csv')\n",
    "test2 = pd.read_csv(path1 + '/test_transaction_round2.csv')\n",
    "test1['day'] = test1['day'].apply(lambda x: x + 64)\n",
    "test2['day'] = test2['day'].apply(lambda x: x + 64)\n",
    "result = pd.read_csv(path1 + '/submit_example.csv')\n",
    "\n",
    "operation_data = pd.concat([operation_data, test1], axis=0, ignore_index=True)\n",
    "transaction_data = pd.concat([transaction_data, test2], axis=0, ignore_index=True)\n",
    "data = pd.concat([data, result[['UID']]], axis=0, ignore_index=True)\n",
    "\n",
    "operation_data['hour'] = operation_data['time'].apply(lambda x:int(x.split(':')[0]))\n",
    "transaction_data['hour'] = transaction_data['time'].apply(lambda x:int(x.split(':')[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 去掉channel118\n",
    "data = data[~data.UID.isin(list(set(transaction_data[transaction_data.channel.isin([118])]['UID'])))]\n",
    "operation_data = operation_data[~operation_data.UID.isin(list(set(transaction_data[transaction_data.channel.isin([118])]['UID'])))]\n",
    "transaction_data = transaction_data[~transaction_data.UID.isin(list(set(transaction_data[transaction_data.channel.isin([118])]['UID'])))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 去掉amt_src1 fd4d2d1006a95637\n",
    "data = data[~data.UID.isin(list(set(transaction_data[transaction_data['amt_src1'].isin(['fd4d2d1006a95637'])]['UID'])))]\n",
    "operation_data = operation_data[~operation_data.UID.isin(list(set(transaction_data[transaction_data['amt_src1'].isin(['fd4d2d1006a95637'])]['UID'])))]\n",
    "transaction_data = transaction_data[~transaction_data.UID.isin(list(set(transaction_data[transaction_data['amt_src1'].isin(['fd4d2d1006a95637'])]['UID'])))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 去掉code1 f1fa4af14fd5b68f\n",
    "data = data[~data.UID.isin(list(set(transaction_data[transaction_data['code1'].isin(['f1fa4af14fd5b68f'])]['UID'])))]\n",
    "operation_data = operation_data[~operation_data.UID.isin(list(set(transaction_data[transaction_data['code1'].isin(['f1fa4af14fd5b68f'])]['UID'])))]\n",
    "transaction_data = transaction_data[~transaction_data.UID.isin(list(set(transaction_data[transaction_data['code1'].isin(['f1fa4af14fd5b68f'])]['UID'])))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "操作表零碎特征：\n",
    "\"\"\"\n",
    "# 每一个用户操作次数\n",
    "operation_count = operation_data[['UID']]\n",
    "operation_count['operation_count'] = 1\n",
    "operation_count = operation_count.groupby('UID').agg('count').reset_index()\n",
    "data = pd.merge(data, operation_count, on='UID', how='left')\n",
    "\n",
    "# 每一个用户交易次数\n",
    "transaction_count = transaction_data[['UID']]\n",
    "transaction_count['transaction_count'] = 1\n",
    "transaction_count = transaction_count.groupby('UID').agg('count').reset_index()\n",
    "data = pd.merge(data, transaction_count, on='UID', how='left')\n",
    "\n",
    "# 01编码：是否有操作或者交易\n",
    "data['has_operation'] = data['operation_count'].apply(lambda x: 1 if x > 0 else 0)\n",
    "data['has_transaction'] = data['transaction_count'].apply(lambda x: 1 if x > 0 else 0)\n",
    "\n",
    "# 交易次数 - 操作次数\n",
    "data['operation_transaction_count_gap'] = data['transaction_count'] - data['operation_count']\n",
    "data = data.drop(['transaction_count', 'operation_count'], axis=1)\n",
    "\n",
    "# 成功次数、成功率\n",
    "success = operation_data[['UID', 'success']]\n",
    "success = success.groupby('UID')['success'].agg('sum').reset_index()\n",
    "all = operation_data[['UID']]\n",
    "all['all'] = 1\n",
    "all = all.groupby('UID').agg('count').reset_index()\n",
    "all = pd.merge(all, success, on='UID', how='left')\n",
    "all['success_rate'] = all['success'] / all['all']\n",
    "all = all[['UID', 'success_rate']]\n",
    "data = pd.merge(data, all, on='UID', how='left')\n",
    "\n",
    "# 每一种操作类型的成功率\n",
    "mode_success = operation_data[['UID', 'mode', 'success']]\n",
    "mode_success = mode_success.groupby(['UID', 'mode'])['success'].agg('sum').reset_index()\n",
    "mode_all = operation_data[['UID', 'mode']]\n",
    "mode_all['mode_all'] = 1\n",
    "mode_all = mode_all.groupby(['UID', 'mode']).agg('count').reset_index()\n",
    "mode_all = pd.merge(mode_all, mode_success, on=['UID', 'mode'], how='left')\n",
    "mode_all['success_rate'] = mode_all['success'] / mode_all['mode_all']\n",
    "mode_all = mode_all[['UID', 'success_rate']]\n",
    "mode_all = mode_all.groupby('UID')['success_rate'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "mode_all.columns = ['UID', 'mode_success_max', 'mode_success_std', 'mode_success_sum', 'mode_success_min', 'mode_success_mean']\n",
    "data = pd.merge(data, mode_all, on='UID', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_nunique(data, table, x, t):\n",
    "    \"\"\"\n",
    "    每一个UID字段的x种类数\n",
    "    table: 交易表或者操作表\n",
    "    x：字段名\n",
    "    \"\"\"\n",
    "    temp = table[['UID', x]]\n",
    "    temp[x + '_nunique_of_' + t] = temp[x]\n",
    "    temp = temp.groupby('UID')[x + '_nunique_of_' + t].agg('nunique').reset_index()\n",
    "    data = pd.merge(data, temp, on='UID', how='left')\n",
    "    return data\n",
    "\n",
    "operation_label = ['day', 'mode', 'os', 'version', 'device1', 'device2', 'mac1', 'ip1', 'ip2', 'device_code3',\n",
    "                   'mac2', 'wifi', 'geo_code', 'ip1_sub', 'ip2_sub'] \n",
    "transaction_label = ['day', 'channel', 'amt_src1', 'merchant', 'code1', 'code2', 'trans_type1', 'acc_id1', 'device_code1',\n",
    "                     'device_code2', 'device_code3', 'device1', 'device2', 'mac1', 'ip1', 'amt_src2', 'acc_id2', 'acc_id3', 'geo_code', 'trans_type2', 'market_code', 'market_type', 'ip1_sub'] \n",
    "for x in operation_label: \n",
    "    data = get_nunique(data, operation_data, x, 'operation') \n",
    "for x in transaction_label: \n",
    "    data = get_nunique(data, transaction_data, x, 'transaction')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_time_frequence(data, table):  \n",
    "    \"\"\"\n",
    "    用户级：用户时间频率\n",
    "    \"\"\"\n",
    "    # 每一个用户每一天每一小时操作数的最大值，最小值，均值，标准差\n",
    "    frequence_one_hour = table[['UID', 'day', 'hour']]\n",
    "    frequence_one_hour['everyday_everyhour'] = 1\n",
    "    frequence_one_hour = frequence_one_hour.groupby(['UID', 'day', 'hour']).agg('count').reset_index()\n",
    "    frequence_one_hour = frequence_one_hour.drop(['day', 'hour'],axis = 1)\n",
    "    frequence_one_hour = frequence_one_hour.groupby('UID')['everyday_everyhour'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    frequence_one_hour.columns = ['UID', 'frequence_one_hour_mean', 'frequence_one_hour_max', 'frequence_one_hour_min', 'frequence_one_hour_std']\n",
    "    data = pd.merge(data, frequence_one_hour, on='UID', how='left')\n",
    "\n",
    "    # 每一个用户每一天操作数的最大值，最小值，均值，标准差\n",
    "    frequence_one_day = table[['UID', 'day']]\n",
    "    frequence_one_day['everyday'] = 1\n",
    "    frequence_one_day = frequence_one_day.groupby(['UID', 'day']).agg('count').reset_index()\n",
    "    frequence_one_day = frequence_one_day.drop('day', axis=1)\n",
    "    frequence_one_day = frequence_one_day.groupby('UID')['everyday'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    frequence_one_day.columns = ['UID', 'frequence_one_day_mean', 'frequence_one_day_max', 'frequence_one_day_min', 'frequence_one_day_std']\n",
    "    data = pd.merge(data, frequence_one_day, on='UID', how='left')\n",
    "\n",
    "    # 每一个用户每一天半夜、早上、下午、晚上的最大值，最小值，均值，标准差\n",
    "    frequence_one_hour = table[['UID', 'day', 'hour']]\n",
    "    frequence_one_hour['everyday_everyhour'] = 1\n",
    "    frequence_one_hour = frequence_one_hour.groupby(['UID', 'day', 'hour']).agg('count').reset_index()\n",
    "\n",
    "    frequence_morning = frequence_one_hour[frequence_one_hour.hour <= 5][['UID', 'day', 'everyday_everyhour']]\n",
    "    frequence_morning['everyday_morning'] = frequence_morning['everyday_everyhour']\n",
    "    frequence_morning = frequence_morning.groupby(['UID', 'day'])['everyday_morning'].agg('sum').reset_index()\n",
    "    frequence_morning = frequence_morning[['UID', 'everyday_morning']]\n",
    "    frequence_morning = frequence_morning.groupby('UID')['everyday_morning'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    frequence_morning.columns = ['UID', 'frequence_morning_mean', 'frequence_morning_max', 'frequence_morning_min', 'frequence_morning_std']\n",
    "    data = pd.merge(data, frequence_morning, on='UID', how='left')\n",
    "\n",
    "    frequence_work_time = frequence_one_hour[(frequence_one_hour.hour <= 11)&(frequence_one_hour.hour >= 6)][['UID', 'day', 'everyday_everyhour']]\n",
    "    frequence_work_time['everyday_work_time'] = frequence_work_time['everyday_everyhour']\n",
    "    frequence_work_time = frequence_work_time.groupby(['UID', 'day'])['everyday_work_time'].agg('sum').reset_index()\n",
    "    frequence_work_time = frequence_work_time[['UID', 'everyday_work_time']]\n",
    "    frequence_work_time = frequence_work_time.groupby('UID')['everyday_work_time'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    frequence_work_time.columns = ['UID', 'frequence_work_time_mean', 'frequence_work_time_max', 'frequence_work_time_min', 'frequence_work_time_std']\n",
    "    data = pd.merge(data, frequence_work_time, on='UID', how='left')\n",
    "\n",
    "    frequence_afternoon = frequence_one_hour[(frequence_one_hour.hour <= 17)&(frequence_one_hour.hour >= 12)][['UID', 'day', 'everyday_everyhour']]\n",
    "    frequence_afternoon['everyday_afternoon'] = frequence_afternoon['everyday_everyhour']\n",
    "    frequence_afternoon = frequence_afternoon.groupby(['UID', 'day'])['everyday_afternoon'].agg('sum').reset_index()\n",
    "    frequence_afternoon = frequence_afternoon[['UID', 'everyday_afternoon']]\n",
    "    frequence_afternoon = frequence_afternoon.groupby('UID')['everyday_afternoon'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    frequence_afternoon.columns = ['UID', 'frequence_afternoon_mean', 'frequence_afternoon_max', 'frequence_afternoon_min', 'frequence_afternoon_std']\n",
    "    data = pd.merge(data, frequence_afternoon, on='UID', how='left')\n",
    "\n",
    "    frequence_night = frequence_one_hour[(frequence_one_hour.hour <= 23)&(frequence_one_hour.hour >= 18)][['UID', 'day', 'everyday_everyhour']]\n",
    "    frequence_night['everyday_night'] = frequence_night['everyday_everyhour']\n",
    "    frequence_night = frequence_night.groupby(['UID', 'day'])['everyday_night'].agg('sum').reset_index()\n",
    "    frequence_night = frequence_night[['UID', 'everyday_night']]\n",
    "    frequence_night = frequence_night.groupby('UID')['everyday_night'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    frequence_night.columns = ['UID', 'frequence_night_mean', 'frequence_night_max', 'frequence_night_min', 'frequence_night_std']\n",
    "    data = pd.merge(data, frequence_night, on='UID', how='left')\n",
    "\n",
    "    # 每一个用户这一天距离前面一段时间操作数的最大值，最小值，均值，标准差\n",
    "    frequence_one_day_gap = table[['UID', 'day']]\n",
    "    frequence_one_day_gap['everyday'] = 1\n",
    "    frequence_one_day_gap = frequence_one_day_gap.groupby(['UID', 'day']).agg('count').reset_index()\n",
    "    frequence_one_day_gap['everyday_before'] = frequence_one_day_gap.groupby('UID')['everyday'].shift(1)\n",
    "    frequence_one_day_gap['everyday_before_gap'] = frequence_one_day_gap['everyday'] - frequence_one_day_gap['everyday_before']\n",
    "    frequence_one_day_gap = frequence_one_day_gap[['UID', 'everyday_before_gap']].groupby('UID')['everyday_before_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    frequence_one_day_gap.columns = ['UID', 'frequence_one_day_gap_sum', 'frequence_one_day_gap_mean', 'frequence_one_day_gap_max', 'frequence_one_day_gap_min', 'frequence_one_day_gap_std']\n",
    "    data = pd.merge(data, frequence_one_day_gap, on='UID', how='left')\n",
    "\n",
    "    # 每一个用户这一小时距离前面一段时间操作数的最大值，最小值，均值，标准差\n",
    "    frequence_one_hour_gap = table[['UID', 'day', 'hour']]\n",
    "    frequence_one_hour_gap['everyhour'] = 1\n",
    "    frequence_one_hour_gap = frequence_one_hour_gap.groupby(['UID', 'day', 'hour']).agg('count').reset_index()\n",
    "    frequence_one_hour_gap['everyhour_before'] = frequence_one_hour_gap.groupby('UID')['everyhour'].shift(1)\n",
    "    frequence_one_hour_gap['everyhour_before_gap'] = frequence_one_hour_gap['everyhour'] - frequence_one_hour_gap['everyhour_before']\n",
    "    frequence_one_hour_gap = frequence_one_hour_gap[['UID', 'everyhour_before_gap']].groupby('UID')['everyhour_before_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    frequence_one_hour_gap.columns = ['UID','frequence_one_hour_gap_sum' ,'frequence_one_hour_gap_mean', 'frequence_one_hour_gap_max', 'frequence_one_hour_gap_min', 'frequence_one_hour_gap_std']\n",
    "    data = pd.merge(data, frequence_one_hour_gap, on='UID', how='left')\n",
    "\n",
    "    # 每一个用户这一天距离前面一段时间操作数的比率和\n",
    "    frequence_one_day_rate = table[['UID', 'day']]\n",
    "    frequence_one_day_rate['everyday'] = 1\n",
    "    frequence_one_day_rate = frequence_one_day_rate.groupby(['UID', 'day']).agg('count').reset_index()\n",
    "    frequence_one_day_rate['everyday_before'] = frequence_one_day_rate.groupby('UID')['everyday'].shift(1)\n",
    "    frequence_one_day_rate['everyday_before_rate'] = frequence_one_day_rate['everyday'] / frequence_one_day_rate['everyday_before']\n",
    "    frequence_one_day_rate = frequence_one_day_rate[['UID', 'everyday_before_rate']].groupby('UID')['everyday_before_rate'].agg('sum').reset_index()\n",
    "    data = pd.merge(data, frequence_one_day_rate, on='UID', how='left')\n",
    "\n",
    "    # 每一个用户这一小时距离前面一段时间操作数的比率和\n",
    "    frequence_one_hour_rate = table[['UID', 'day', 'hour']]\n",
    "    frequence_one_hour_rate['everyhour'] = 1\n",
    "    frequence_one_hour_rate = frequence_one_hour_rate.groupby(['UID', 'day', 'hour']).agg('count').reset_index()\n",
    "    frequence_one_hour_rate['everyhour_before'] = frequence_one_hour_rate.groupby('UID')['everyhour'].shift(1)\n",
    "    frequence_one_hour_rate['everyhour_before_rate'] = frequence_one_hour_rate['everyhour'] - frequence_one_hour_rate['everyhour_before']\n",
    "    frequence_one_hour_rate = frequence_one_hour_rate[['UID', 'everyhour_before_rate']].groupby('UID')['everyhour_before_rate'].agg('sum').reset_index()\n",
    "    data = pd.merge(data, frequence_one_hour_rate, on='UID', how='left')\n",
    "\n",
    "    # 每一个用户每三天的操作量的最大值、最小值、均值、标准差\n",
    "    frequence_three_day = table[['UID', 'day']]\n",
    "    frequence_three_day['three_day'] = 1\n",
    "    frequence_three_day = frequence_three_day.groupby(['UID', 'day']).agg('count').reset_index()\n",
    "    frequence_three_day['before_three_day'] = frequence_three_day.groupby('UID')['three_day'].shift(3)\n",
    "    frequence_three_day['three_day_gap'] = frequence_three_day['before_three_day'] - frequence_three_day['three_day']\n",
    "    frequence_three_day = frequence_three_day[['UID', 'three_day_gap']]\n",
    "    frequence_three_day = frequence_three_day.groupby('UID')['three_day_gap'].agg({'sum', 'max', 'min', 'mean', 'std'}).reset_index()\n",
    "    frequence_three_day.columns = ['UID', 'frequence_three_day_sum', 'frequence_three_day_mean', 'frequence_three_day_max', 'frequence_three_day_min', 'frequence_three_day_std']\n",
    "    data = pd.merge(data, frequence_three_day, on='UID', how='left')\n",
    "\n",
    "    # 每一个用户每七天的操作量的最大值、最小值、均值、标准差\n",
    "    frequence_sixteen_day = table[['UID', 'day']]\n",
    "    frequence_sixteen_day['sixteen_day'] = 1\n",
    "    frequence_sixteen_day = frequence_sixteen_day.groupby(['UID', 'day']).agg('count').reset_index()\n",
    "    frequence_sixteen_day['before_sixteen_day'] = frequence_sixteen_day.groupby('UID')['sixteen_day'].shift(7)\n",
    "    frequence_sixteen_day['sixteen_day_gap'] = frequence_sixteen_day['before_sixteen_day'] - frequence_sixteen_day['sixteen_day']\n",
    "    frequence_sixteen_day = frequence_sixteen_day[['UID', 'sixteen_day_gap']]\n",
    "    frequence_sixteen_day = frequence_sixteen_day.groupby('UID')['sixteen_day_gap'].agg({'sum', 'max', 'min', 'mean', 'std'}).reset_index()\n",
    "    frequence_sixteen_day.columns = ['UID', 'frequence_sixteen_day_sum', 'frequence_sixteen_day_mean', 'frequence_sixteen_day_max', 'frequence_sixteen_day_min', 'frequence_sixteen_day_std']\n",
    "    data = pd.merge(data, frequence_sixteen_day, on='UID', how='left')\n",
    "\n",
    "    # 每一个用户每三天的操作量的最大值、最小值、均值、标准差比率和\n",
    "    frequence_three_day = table[['UID', 'day']]\n",
    "    frequence_three_day['three_day'] = 1\n",
    "    frequence_three_day = frequence_three_day.groupby(['UID', 'day']).agg('count').reset_index()\n",
    "    frequence_three_day['before_three_day'] = frequence_three_day.groupby('UID')['three_day'].shift(3)\n",
    "    frequence_three_day['three_day_rate'] = frequence_three_day['three_day'] / frequence_three_day['before_three_day']\n",
    "    frequence_three_day = frequence_three_day[['UID', 'three_day_rate']]\n",
    "    frequence_three_day = frequence_three_day.groupby('UID')['three_day_rate'].agg('sum').reset_index()\n",
    "    data = pd.merge(data, frequence_three_day, on='UID', how='left')\n",
    "\n",
    "    # 每一个用户每七天的操作量的最大值、最小值、均值、标准差比率和\n",
    "    frequence_sixteen_day = table[['UID', 'day']]\n",
    "    frequence_sixteen_day['sixteen_day'] = 1\n",
    "    frequence_sixteen_day = frequence_sixteen_day.groupby(['UID', 'day']).agg('count').reset_index()\n",
    "    frequence_sixteen_day['before_sixteen_day'] = frequence_sixteen_day.groupby('UID')['sixteen_day'].shift(7)\n",
    "    frequence_sixteen_day['sixteen_day_rate'] = frequence_sixteen_day['sixteen_day'] / frequence_sixteen_day['before_sixteen_day']\n",
    "    frequence_sixteen_day = frequence_sixteen_day[['UID', 'sixteen_day_rate']]\n",
    "    frequence_sixteen_day = frequence_sixteen_day.groupby('UID')['sixteen_day_rate'].agg('sum').reset_index()\n",
    "    data = pd.merge(data, frequence_sixteen_day, on='UID', how='left')\n",
    "\n",
    "    # 每秒用户的操作最大值、最小值、均值、方差\n",
    "    every_second = table[['UID', 'time']]\n",
    "    every_second['operation_second_caozuo'] = 1\n",
    "    every_second = every_second.groupby(['UID', 'time']).agg('count').reset_index()\n",
    "    every_second = every_second[['UID', 'operation_second_caozuo']]\n",
    "    every_second = every_second.groupby('UID')['operation_second_caozuo'].agg({'max', 'min', 'mean', 'std'}).reset_index()\n",
    "    every_second.columns = ['UID', 'every_second_mean', 'every_second_max', 'every_second_min', 'every_second_std']\n",
    "    data = pd.merge(data, every_second, on='UID', how='left')\n",
    "\n",
    "    # 每分钟用户操作的最大值、最小值、均值、方差\n",
    "    every_minute = table[['UID', 'time']]\n",
    "    every_minute['every_minute'] = every_minute['time'].apply(lambda x: str(int(x.split(':')[0])) + str(int(x.split(':')[1])))\n",
    "    every_minute = every_minute[['UID', 'every_minute']]\n",
    "    every_minute['operation_minute_caozuo'] = 1\n",
    "    every_minute = every_minute.groupby(['UID', 'every_minute']).agg('count').reset_index()\n",
    "    every_minute = every_minute[['UID', 'operation_minute_caozuo']]\n",
    "    every_minute = every_minute.groupby('UID')['operation_minute_caozuo'].agg({'max', 'min', 'mean', 'std'}).reset_index()\n",
    "    every_minute.columns = ['UID', 'every_minute_mean', 'every_minute_max', 'every_minute_min', 'every_minute_std']\n",
    "    data = pd.merge(data, every_minute, on='UID', how='left')\n",
    "\n",
    "    return data\n",
    "\n",
    "data = get_time_frequence(data, operation_data)\n",
    "data = get_time_frequence(data, transaction_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "用户级：用户交易金额时间频率\n",
    "\"\"\"\n",
    "# 用户每天的每小时交易金额的和、均值、最大值、最小值、标准差\n",
    "money_frequence_one_hour = transaction_data[['UID', 'day', 'hour', 'trans_amt']]\n",
    "money_frequence_one_hour = money_frequence_one_hour.groupby(['UID', 'day', 'hour'])['trans_amt'].agg('sum').reset_index()\n",
    "money_frequence_one_hour = money_frequence_one_hour[['UID', 'trans_amt']]\n",
    "money_frequence_one_hour['money_frequence_one_hour'] = money_frequence_one_hour['trans_amt']\n",
    "money_frequence_one_hour = money_frequence_one_hour.groupby('UID')['money_frequence_one_hour'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "money_frequence_one_hour.columns = ['UID', 'money_frequence_one_hour_max', 'money_frequence_one_hour_std', 'money_frequence_one_hour_sum', 'money_frequence_one_hour_min', 'money_frequence_one_hour_mean']\n",
    "data = pd.merge(data, money_frequence_one_hour, on='UID', how='left')\n",
    "\n",
    "# 用户每天的每小时交易金额的上下两段时间的增量\n",
    "money_frequence_one_hour_gap = transaction_data[['UID', 'day', 'hour', 'trans_amt']]\n",
    "money_frequence_one_hour_gap = money_frequence_one_hour_gap.groupby(['UID', 'day', 'hour'])['trans_amt'].agg('sum').reset_index()\n",
    "money_frequence_one_hour_gap['before_hour_amt'] = money_frequence_one_hour_gap.groupby('UID')['trans_amt'].shift(1)\n",
    "money_frequence_one_hour_gap['money_frequence_one_hour_gap'] = money_frequence_one_hour_gap['trans_amt'] - money_frequence_one_hour_gap['before_hour_amt']\n",
    "money_frequence_one_hour_gap = money_frequence_one_hour_gap[['UID', 'money_frequence_one_hour_gap']]\n",
    "money_frequence_one_hour_gap = money_frequence_one_hour_gap.groupby('UID')['money_frequence_one_hour_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "money_frequence_one_hour_gap.columns = ['UID', 'money_frequence_one_hour_sum', 'money_frequence_one_hour_max', 'money_frequence_one_hour_std', 'money_frequence_one_hour_min', 'money_frequence_one_hour_mean']\n",
    "data = pd.merge(data, money_frequence_one_hour_gap, on='UID', how='left')\n",
    "\n",
    "# 用户每天的每小时交易金额的上下两段时间的比率和\n",
    "money_frequence_one_hour_rate = transaction_data[['UID', 'day', 'hour', 'trans_amt']]\n",
    "money_frequence_one_hour_rate = money_frequence_one_hour_rate.groupby(['UID', 'day', 'hour'])['trans_amt'].agg('sum').reset_index()\n",
    "money_frequence_one_hour_rate['before_hour_amt'] = money_frequence_one_hour_rate.groupby('UID')['trans_amt'].shift(1)\n",
    "money_frequence_one_hour_rate['money_frequence_one_hour_rate'] = money_frequence_one_hour_rate['trans_amt'] / money_frequence_one_hour_rate['before_hour_amt']\n",
    "money_frequence_one_hour_rate = money_frequence_one_hour_rate[['UID', 'money_frequence_one_hour_rate']]\n",
    "money_frequence_one_hour_rate = money_frequence_one_hour_rate.groupby('UID')['money_frequence_one_hour_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, money_frequence_one_hour_rate, on='UID', how='left')\n",
    "\n",
    "\n",
    "# 用户每天的五个小时的和、均值、最大值、最小值、标准差(半夜、早上、下午、晚上)，gap、rate\n",
    "# for morning\n",
    "money_frequence_morning = transaction_data[transaction_data.hour <= 5][['UID', 'day', 'trans_amt']]\n",
    "money_frequence_morning = money_frequence_morning.groupby(['UID', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "money_frequence_morning = money_frequence_morning[['UID', 'trans_amt']]\n",
    "money_frequence_morning['trans_amt_sum'] = money_frequence_morning['trans_amt']\n",
    "money_frequence_morning = money_frequence_morning.groupby('UID')['trans_amt_sum'].agg({'sum', 'mean', 'max', 'min', 'mean', 'std'}).reset_index()\n",
    "money_frequence_morning.columns = ['UID', 'money_frequence_morning_max', 'money_frequence_morning_std', 'money_frequence_morning_sum', 'money_frequence_morning_min', 'money_frequence_morning_mean']\n",
    "data = pd.merge(data, money_frequence_morning, on='UID', how='left')\n",
    "\n",
    "money_frequence_morning_gap = transaction_data[transaction_data.hour <= 5][['UID', 'day', 'trans_amt']]\n",
    "money_frequence_morning_gap = money_frequence_morning_gap.groupby(['UID', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "money_frequence_morning_gap = money_frequence_morning_gap[['UID', 'trans_amt']]\n",
    "money_frequence_morning_gap['trans_amt_before'] = money_frequence_morning_gap.groupby('UID')['trans_amt'].shift(1)\n",
    "money_frequence_morning_gap['trans_amt_gap'] = money_frequence_morning_gap['trans_amt'] - money_frequence_morning_gap['trans_amt_before']\n",
    "money_frequence_morning_gap = money_frequence_morning_gap[['UID', 'trans_amt_gap']]\n",
    "money_frequence_morning_gap = money_frequence_morning_gap.groupby('UID')['trans_amt_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "money_frequence_morning_gap.columns = ['UID', 'money_frequence_morning_gap_sum', 'money_frequence_morning_gap_max', 'money_frequence_morning_gap_std', 'money_frequence_morning_gap_min', 'money_frequence_morning_gap_mean']\n",
    "data = pd.merge(data, money_frequence_morning_gap, on='UID', how='left')\n",
    "\n",
    "money_frequence_morning_rate = transaction_data[transaction_data.hour <= 5][['UID', 'day', 'trans_amt']]\n",
    "money_frequence_morning_rate = money_frequence_morning_rate.groupby(['UID', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "money_frequence_morning_rate = money_frequence_morning_rate[['UID', 'trans_amt']]\n",
    "money_frequence_morning_rate['trans_amt_before'] = money_frequence_morning_rate.groupby('UID')['trans_amt'].shift(1)\n",
    "money_frequence_morning_rate['trans_amt_rate'] = money_frequence_morning_rate['trans_amt'] / money_frequence_morning_rate['trans_amt_before']\n",
    "money_frequence_morning_rate = money_frequence_morning_rate[['UID', 'trans_amt_rate']]\n",
    "money_frequence_morning_rate = money_frequence_morning_rate.groupby('UID')['trans_amt_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, money_frequence_morning_rate, on='UID', how='left')\n",
    "\n",
    "# for work_time\n",
    "money_frequence_work_time = transaction_data[(transaction_data.hour >=6)&(transaction_data.hour <= 11)][['UID', 'day', 'trans_amt']]\n",
    "money_frequence_work_time = money_frequence_work_time.groupby(['UID', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "money_frequence_work_time = money_frequence_work_time[['UID', 'trans_amt']]\n",
    "money_frequence_work_time['trans_amt_sum'] = money_frequence_work_time['trans_amt']\n",
    "money_frequence_work_time = money_frequence_work_time.groupby('UID')['trans_amt_sum'].agg({'sum', 'mean', 'max', 'min', 'mean', 'std'}).reset_index()\n",
    "money_frequence_work_time.columns = ['UID', 'money_frequence_work_time_max', 'money_frequence_work_time_std', 'money_frequence_work_time_sum', 'money_frequence_work_time_min', 'money_frequence_work_time_mean']\n",
    "data = pd.merge(data, money_frequence_work_time, on='UID', how='left')\n",
    "\n",
    "money_frequence_work_time_gap = transaction_data[(transaction_data.hour >=6)&(transaction_data.hour <= 11)][['UID', 'day', 'trans_amt']]\n",
    "money_frequence_work_time_gap = money_frequence_work_time_gap.groupby(['UID', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "money_frequence_work_time_gap = money_frequence_work_time_gap[['UID', 'trans_amt']]\n",
    "money_frequence_work_time_gap['trans_amt_before'] = money_frequence_work_time_gap.groupby('UID')['trans_amt'].shift(1)\n",
    "money_frequence_work_time_gap['trans_amt_gap'] = money_frequence_work_time_gap['trans_amt'] - money_frequence_work_time_gap['trans_amt_before']\n",
    "money_frequence_work_time_gap = money_frequence_work_time_gap[['UID', 'trans_amt_gap']]\n",
    "money_frequence_work_time_gap = money_frequence_work_time_gap.groupby('UID')['trans_amt_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "money_frequence_work_time_gap.columns = ['UID', 'money_frequence_work_time_gap_sum', 'money_frequence_work_time_gap_max', 'money_frequence_work_time_gap_std', 'money_frequence_work_time_gap_min', 'money_frequence_work_time_gap_mean']\n",
    "data = pd.merge(data, money_frequence_work_time_gap, on='UID', how='left')\n",
    "\n",
    "money_frequence_work_time_rate = transaction_data[(transaction_data.hour >=6)&(transaction_data.hour <= 11)][['UID', 'day', 'trans_amt']]\n",
    "money_frequence_work_time_rate = money_frequence_work_time_rate.groupby(['UID', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "money_frequence_work_time_rate = money_frequence_work_time_rate[['UID', 'trans_amt']]\n",
    "money_frequence_work_time_rate['trans_amt_before'] = money_frequence_work_time_rate.groupby('UID')['trans_amt'].shift(1)\n",
    "money_frequence_work_time_rate['trans_amt_rate'] = money_frequence_work_time_rate['trans_amt'] / money_frequence_work_time_rate['trans_amt_before']\n",
    "money_frequence_work_time_rate = money_frequence_work_time_rate[['UID', 'trans_amt_rate']]\n",
    "money_frequence_work_time_rate = money_frequence_work_time_rate.groupby('UID')['trans_amt_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, money_frequence_work_time_rate, on='UID', how='left')\n",
    "\n",
    "# for afternoon\n",
    "money_frequence_afternoon = transaction_data[(transaction_data.hour >=12)&(transaction_data.hour <= 17)][['UID', 'day', 'trans_amt']]\n",
    "money_frequence_afternoon = money_frequence_afternoon.groupby(['UID', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "money_frequence_afternoon = money_frequence_afternoon[['UID', 'trans_amt']]\n",
    "money_frequence_afternoon['trans_amt_sum'] = money_frequence_afternoon['trans_amt']\n",
    "money_frequence_afternoon = money_frequence_afternoon.groupby('UID')['trans_amt_sum'].agg({'sum', 'mean', 'max', 'min', 'mean', 'std'}).reset_index()\n",
    "money_frequence_afternoon.columns = ['UID', 'money_frequence_afternoon_max', 'money_frequence_afternoon_std', 'money_frequence_afternoon_sum', 'money_frequence_afternoon_min', 'money_frequence_afternoon_mean']\n",
    "data = pd.merge(data, money_frequence_afternoon, on='UID', how='left')\n",
    "\n",
    "money_frequence_afternoon_gap = transaction_data[(transaction_data.hour >=12)&(transaction_data.hour <= 17)][['UID', 'day', 'trans_amt']]\n",
    "money_frequence_afternoon_gap = money_frequence_afternoon_gap.groupby(['UID', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "money_frequence_afternoon_gap = money_frequence_afternoon_gap[['UID', 'trans_amt']]\n",
    "money_frequence_afternoon_gap['trans_amt_before'] = money_frequence_afternoon_gap.groupby('UID')['trans_amt'].shift(1)\n",
    "money_frequence_afternoon_gap['trans_amt_gap'] = money_frequence_afternoon_gap['trans_amt'] - money_frequence_afternoon_gap['trans_amt_before']\n",
    "money_frequence_afternoon_gap = money_frequence_afternoon_gap[['UID', 'trans_amt_gap']]\n",
    "money_frequence_afternoon_gap = money_frequence_afternoon_gap.groupby('UID')['trans_amt_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "money_frequence_afternoon_gap.columns = ['UID', 'money_frequence_afternoon_gap_sum', 'money_frequence_afternoon_gap_max', 'money_frequence_afternoon_gap_std', 'money_frequence_afternoon_gap_min', 'money_frequence_afternoon_gap_mean']\n",
    "data = pd.merge(data, money_frequence_afternoon_gap, on='UID', how='left')\n",
    "\n",
    "money_frequence_afternoon_rate = transaction_data[(transaction_data.hour >=12)&(transaction_data.hour <= 17)][['UID', 'day', 'trans_amt']]\n",
    "money_frequence_afternoon_rate = money_frequence_afternoon_rate.groupby(['UID', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "money_frequence_afternoon_rate = money_frequence_afternoon_rate[['UID', 'trans_amt']]\n",
    "money_frequence_afternoon_rate['trans_amt_before'] = money_frequence_afternoon_rate.groupby('UID')['trans_amt'].shift(1)\n",
    "money_frequence_afternoon_rate['trans_amt_rate'] = money_frequence_afternoon_rate['trans_amt'] / money_frequence_afternoon_rate['trans_amt_before']\n",
    "money_frequence_afternoon_rate = money_frequence_afternoon_rate[['UID', 'trans_amt_rate']]\n",
    "money_frequence_afternoon_rate = money_frequence_afternoon_rate.groupby('UID')['trans_amt_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, money_frequence_afternoon_rate, on='UID', how='left')\n",
    "\n",
    "# for night\n",
    "money_frequence_night = transaction_data[(transaction_data.hour >=18)&(transaction_data.hour <= 23)][['UID', 'day', 'trans_amt']]\n",
    "money_frequence_night = money_frequence_night.groupby(['UID', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "money_frequence_night = money_frequence_night[['UID', 'trans_amt']]\n",
    "money_frequence_night['trans_amt_sum'] = money_frequence_night['trans_amt']\n",
    "money_frequence_night = money_frequence_night.groupby('UID')['trans_amt_sum'].agg({'sum', 'mean', 'max', 'min', 'mean', 'std'}).reset_index()\n",
    "money_frequence_night.columns = ['UID', 'money_frequence_night_max', 'money_frequence_night_std', 'money_frequence_night_sum', 'money_frequence_night_min', 'money_frequence_night_mean']\n",
    "data = pd.merge(data, money_frequence_night, on='UID', how='left')\n",
    "\n",
    "money_frequence_night_gap = transaction_data[(transaction_data.hour >=18)&(transaction_data.hour <= 23)][['UID', 'day', 'trans_amt']]\n",
    "money_frequence_night_gap = money_frequence_night_gap.groupby(['UID', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "money_frequence_night_gap = money_frequence_night_gap[['UID', 'trans_amt']]\n",
    "money_frequence_night_gap['trans_amt_before'] = money_frequence_night_gap.groupby('UID')['trans_amt'].shift(1)\n",
    "money_frequence_night_gap['trans_amt_gap'] = money_frequence_night_gap['trans_amt'] - money_frequence_night_gap['trans_amt_before']\n",
    "money_frequence_night_gap = money_frequence_night_gap[['UID', 'trans_amt_gap']]\n",
    "money_frequence_night_gap = money_frequence_night_gap.groupby('UID')['trans_amt_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "money_frequence_night_gap.columns = ['UID', 'money_frequence_night_gap_sum', 'money_frequence_night_gap_max', 'money_frequence_night_gap_std', 'money_frequence_night_gap_min', 'money_frequence_night_gap_mean']\n",
    "data = pd.merge(data, money_frequence_night_gap, on='UID', how='left')\n",
    "\n",
    "money_frequence_night_rate = transaction_data[(transaction_data.hour >=18)&(transaction_data.hour <= 23)][['UID', 'day', 'trans_amt']]\n",
    "money_frequence_night_rate = money_frequence_night_rate.groupby(['UID', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "money_frequence_night_rate = money_frequence_night_rate[['UID', 'trans_amt']]\n",
    "money_frequence_night_rate['trans_amt_before'] = money_frequence_night_rate.groupby('UID')['trans_amt'].shift(1)\n",
    "money_frequence_night_rate['trans_amt_rate'] = money_frequence_night_rate['trans_amt'] / money_frequence_night_rate['trans_amt_before']\n",
    "money_frequence_night_rate = money_frequence_night_rate[['UID', 'trans_amt_rate']]\n",
    "money_frequence_night_rate = money_frequence_night_rate.groupby('UID')['trans_amt_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, money_frequence_night_rate, on='UID', how='left')\n",
    "\n",
    "\n",
    "# 用户每天的交易金额和、最大值、最小值、均值、极差、gap、rate\n",
    "money_frequence_day = transaction_data[['UID', 'day', 'trans_amt']]\n",
    "money_frequence_day = money_frequence_day.groupby(['UID', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "money_frequence_day = money_frequence_day[['UID', 'trans_amt']]\n",
    "money_frequence_day['trans_amt_day'] = money_frequence_day['trans_amt']\n",
    "money_frequence_day = money_frequence_day.groupby('UID')['trans_amt_day'].agg({'max', 'min', 'mean', 'std'}).reset_index()\n",
    "money_frequence_day.columns = ['UID', 'money_frequence_day_max', 'money_frequence_day_std', 'money_frequence_day_min', 'money_frequence_day_mean']\n",
    "data = pd.merge(data, money_frequence_day, on='UID', how='left')\n",
    "\n",
    "money_frequence_day_gap = transaction_data[['UID', 'day', 'trans_amt']]\n",
    "money_frequence_day_gap = money_frequence_day_gap.groupby(['UID', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "money_frequence_day_gap = money_frequence_day_gap[['UID', 'trans_amt']]\n",
    "money_frequence_day_gap['trans_amt_before'] = money_frequence_day_gap.groupby('UID')['trans_amt'].shift(1)\n",
    "money_frequence_day_gap['trans_amt_gap'] = money_frequence_day_gap['trans_amt'] - money_frequence_day_gap['trans_amt_before']\n",
    "money_frequence_day_gap = money_frequence_day_gap[['UID', 'trans_amt_gap']]\n",
    "money_frequence_day_gap = money_frequence_day_gap.groupby('UID')['trans_amt_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "money_frequence_day_gap.columns = ['UID', 'money_frequence_day_gap_sum', 'money_frequence_day_gap_max', 'money_frequence_day_gap_std', 'money_frequence_day_gap_min', 'money_frequence_day_gap_mean']\n",
    "data = pd.merge(data, money_frequence_day_gap, on='UID', how='left')\n",
    "\n",
    "money_frequence_day_rate = transaction_data[['UID', 'day', 'trans_amt']]\n",
    "money_frequence_day_rate = money_frequence_day_rate.groupby(['UID', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "money_frequence_day_rate = money_frequence_day_rate[['UID', 'trans_amt']]\n",
    "money_frequence_day_rate['trans_amt_before'] = money_frequence_day_rate.groupby('UID')['trans_amt'].shift(1)\n",
    "money_frequence_day_rate['trans_amt_rate'] = money_frequence_day_rate['trans_amt'] / money_frequence_day_rate['trans_amt_before']\n",
    "money_frequence_day_rate = money_frequence_day_rate[['UID', 'trans_amt_rate']]\n",
    "money_frequence_day_rate = money_frequence_day_rate.groupby('UID')['trans_amt_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, money_frequence_day_rate, on='UID', how='left')\n",
    "\n",
    "\n",
    "# 每一个用户每三天的交易量的差值的最大值、最小值、均值、标准差\n",
    "money_frequence_three_day = transaction_data[['UID', 'day', 'trans_amt']]\n",
    "money_frequence_three_day = money_frequence_three_day.groupby(['UID', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "money_frequence_three_day['trans_amt_before'] = money_frequence_three_day.groupby('UID')['trans_amt'].shift(3)\n",
    "money_frequence_three_day['trans_amt_three_day_gap'] = money_frequence_three_day['trans_amt'] - money_frequence_three_day['trans_amt_before']\n",
    "money_frequence_three_day = money_frequence_three_day[['UID', 'trans_amt_three_day_gap']]\n",
    "money_frequence_three_day = money_frequence_three_day.groupby('UID')['trans_amt_three_day_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "money_frequence_three_day.columns = ['UID', 'money_frequence_three_day_sum', 'money_frequence_three_day_max', 'money_frequence_three_day_std', 'money_frequence_three_day_min', 'money_frequence_three_day_mean']\n",
    "data = pd.merge(data, money_frequence_three_day, on='UID', how='left')\n",
    "\n",
    "money_frequence_three_day_rate = transaction_data[['UID', 'day', 'trans_amt']]\n",
    "money_frequence_three_day_rate = money_frequence_three_day_rate.groupby(['UID', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "money_frequence_three_day_rate['trans_amt_before'] = money_frequence_three_day_rate.groupby('UID')['trans_amt'].shift(3)\n",
    "money_frequence_three_day_rate['trans_amt_three_day_rate'] = money_frequence_three_day_rate['trans_amt'] / money_frequence_three_day_rate['trans_amt_before']\n",
    "money_frequence_three_day_rate = money_frequence_three_day_rate[['UID', 'trans_amt_three_day_rate']]\n",
    "money_frequence_three_day_rate = money_frequence_three_day_rate.groupby('UID')['trans_amt_three_day_rate'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "money_frequence_three_day_rate.columns = ['UID', 'money_frequence_three_day_rate_max', 'money_frequence_three_day_rate_std', 'money_frequence_three_day_rate_min', 'money_frequence_three_day_rate_mean']\n",
    "data = pd.merge(data, money_frequence_three_day_rate, on='UID', how='left')\n",
    "\n",
    "# 每一个用户每七天的交易量的差值的最大值、最小值、均值、标准差\n",
    "money_frequence_sixteen_day = transaction_data[['UID', 'day', 'trans_amt']]\n",
    "money_frequence_sixteen_day = money_frequence_sixteen_day.groupby(['UID', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "money_frequence_sixteen_day['trans_amt_before'] = money_frequence_sixteen_day.groupby('UID')['trans_amt'].shift(7)\n",
    "money_frequence_sixteen_day['trans_amt_sixteen_day_gap'] = money_frequence_sixteen_day['trans_amt'] - money_frequence_sixteen_day['trans_amt_before']\n",
    "money_frequence_sixteen_day = money_frequence_sixteen_day[['UID', 'trans_amt_sixteen_day_gap']]\n",
    "money_frequence_sixteen_day = money_frequence_sixteen_day.groupby('UID')['trans_amt_sixteen_day_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "money_frequence_sixteen_day.columns = ['UID', 'money_frequence_sixteen_day_sum', 'money_frequence_sixteen_day_max', 'money_frequence_sixteen_day_std', 'money_frequence_sixteen_day_min', 'money_frequence_sixteen_day_mean']\n",
    "data = pd.merge(data, money_frequence_sixteen_day, on='UID', how='left')\n",
    "\n",
    "money_frequence_sixteen_day_rate = transaction_data[['UID', 'day', 'trans_amt']]\n",
    "money_frequence_sixteen_day_rate = money_frequence_sixteen_day_rate.groupby(['UID', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "money_frequence_sixteen_day_rate['trans_amt_before'] = money_frequence_sixteen_day_rate.groupby('UID')['trans_amt'].shift(7)\n",
    "money_frequence_sixteen_day_rate['trans_amt_sixteen_day_rate'] = money_frequence_sixteen_day_rate['trans_amt'] / money_frequence_sixteen_day_rate['trans_amt_before']\n",
    "money_frequence_sixteen_day_rate = money_frequence_sixteen_day_rate[['UID', 'trans_amt_sixteen_day_rate']]\n",
    "money_frequence_sixteen_day_rate = money_frequence_sixteen_day_rate.groupby('UID')['trans_amt_sixteen_day_rate'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "money_frequence_sixteen_day_rate.columns = ['UID', 'money_frequence_sixteen_day_rate_max', 'money_frequence_sixteen_day_rate_std', 'money_frequence_sixteen_day_rate_min', 'money_frequence_sixteen_day_rate_mean']\n",
    "data = pd.merge(data, money_frequence_sixteen_day_rate, on='UID', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "用户级：merchant时间频率\n",
    "\"\"\"\n",
    "# merchant一天之内的总交易额\n",
    "merchant_day = transaction_data[['day', 'merchant', 'trans_amt']]\n",
    "merchant_day = merchant_day.groupby(['merchant', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "\n",
    "merchant_UID = transaction_data[['UID', 'day', 'merchant']]\n",
    "\n",
    "merchant_day = pd.merge(merchant_UID, merchant_day, on=['day', 'merchant'], how='left')\n",
    "merchant_day = merchant_day[['UID', 'trans_amt']]\n",
    "merchant_day['merchant_day_sum'] = merchant_day['trans_amt']\n",
    "merchant_day = merchant_day.groupby('UID')['trans_amt'].agg('sum').reset_index()\n",
    "data = pd.merge(data, merchant_day, on='UID', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "用户 * merchant时间频率\n",
    "\"\"\"\n",
    "# 用户merchant每天每小时的统计\n",
    "merchant_every_hour = transaction_data[['UID', 'merchant', 'day', 'hour', 'trans_amt']]\n",
    "merchant_every_hour = merchant_every_hour.groupby(['UID', 'merchant', 'day', 'hour'])['trans_amt'].agg('sum').reset_index()\n",
    "merchant_every_hour = merchant_every_hour[['UID', 'trans_amt']]\n",
    "merchant_every_hour = merchant_every_hour.groupby('UID')['trans_amt'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "merchant_every_hour.columns = ['UID', 'merchant_every_hour_std', 'merchant_every_hour_mean', 'merchant_every_hour_min', 'merchant_every_hour_max']\n",
    "data = pd.merge(data, merchant_every_hour, on='UID', how='left')\n",
    "\n",
    "# 用户merchant每天每小时的gap\n",
    "merchant_every_hour_gap = transaction_data[['UID', 'merchant', 'day', 'hour', 'trans_amt']]\n",
    "merchant_every_hour_gap = merchant_every_hour_gap.groupby(['UID', 'merchant', 'day', 'hour'])['trans_amt'].agg('sum').reset_index()\n",
    "merchant_every_hour_gap = merchant_every_hour_gap[['UID', 'trans_amt']]\n",
    "merchant_every_hour_gap['trans_amt_before'] = merchant_every_hour_gap.groupby('UID')['trans_amt'].shift(1)\n",
    "merchant_every_hour_gap['trans_amt_hour_gap'] = merchant_every_hour_gap['trans_amt'] - merchant_every_hour_gap['trans_amt_before']\n",
    "merchant_every_hour_gap = merchant_every_hour_gap[['UID', 'trans_amt_hour_gap']]\n",
    "merchant_every_hour_gap = merchant_every_hour_gap.groupby('UID')['trans_amt_hour_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "merchant_every_hour_gap.columns = ['UID', 'merchant_every_hour_gap_sum', 'merchant_every_hour_gap_std', 'merchant_every_hour_gap_mean', 'merchant_every_hour_gap_min', 'merchant_every_hour_gap_max']\n",
    "data = pd.merge(data, merchant_every_hour_gap, on='UID', how='left')\n",
    "\n",
    "# 用户merchant每天每小时的比率和\n",
    "merchant_every_hour_rate = transaction_data[['UID', 'merchant', 'day', 'hour', 'trans_amt']]\n",
    "merchant_every_hour_rate = merchant_every_hour_rate.groupby(['UID', 'merchant', 'day', 'hour'])['trans_amt'].agg('sum').reset_index()\n",
    "merchant_every_hour_rate = merchant_every_hour_rate[['UID', 'trans_amt']]\n",
    "merchant_every_hour_rate['trans_amt_before'] = merchant_every_hour_rate.groupby('UID')['trans_amt'].shift(1)\n",
    "merchant_every_hour_rate['trans_amt_hour_rate'] = merchant_every_hour_rate['trans_amt'] / merchant_every_hour_rate['trans_amt_before']\n",
    "merchant_every_hour_rate = merchant_every_hour_rate[['UID', 'trans_amt_hour_rate']]\n",
    "merchant_every_hour_rate = merchant_every_hour_rate.groupby('UID')['trans_amt_hour_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, merchant_every_hour_rate, on='UID', how='left')\n",
    "\n",
    "\n",
    "# 用户merchant每天的统计\n",
    "merchant_every_day = transaction_data[['UID', 'merchant', 'day', 'trans_amt']]\n",
    "merchant_every_day = merchant_every_day.groupby(['UID', 'merchant', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "merchant_every_day = merchant_every_day[['UID', 'trans_amt']]\n",
    "merchant_every_day = merchant_every_day.groupby('UID')['trans_amt'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "merchant_every_day.columns = ['UID', 'merchant_every_day_std', 'merchant_every_day_mean', 'merchant_every_day_min', 'merchant_every_day_max']\n",
    "data = pd.merge(data, merchant_every_day, on='UID', how='left')\n",
    "\n",
    "# 用户merchant每天的gap\n",
    "merchant_every_day_gap = transaction_data[['UID', 'merchant', 'day', 'trans_amt']]\n",
    "merchant_every_day_gap = merchant_every_day_gap.groupby(['UID', 'merchant', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "merchant_every_day_gap = merchant_every_day_gap[['UID', 'trans_amt']]\n",
    "merchant_every_day_gap['trans_amt_before'] = merchant_every_day_gap.groupby('UID')['trans_amt'].shift(1)\n",
    "merchant_every_day_gap['trans_amt_day_gap'] = merchant_every_day_gap['trans_amt'] - merchant_every_day_gap['trans_amt_before']\n",
    "merchant_every_day_gap = merchant_every_day_gap[['UID', 'trans_amt_day_gap']]\n",
    "merchant_every_day_gap = merchant_every_day_gap.groupby('UID')['trans_amt_day_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "merchant_every_day_gap.columns = ['UID', 'merchant_every_day_gap_sum', 'merchant_every_day_gap_mean', 'merchant_every_day_gap_max', 'merchant_every_day_gap_min', 'merchant_every_day_gap_std']\n",
    "data = pd.merge(data, merchant_every_day_gap, on='UID', how='left')\n",
    "\n",
    "# 用户merchant每天的rate\n",
    "merchant_every_day_rate = transaction_data[['UID', 'merchant', 'day', 'trans_amt']]\n",
    "merchant_every_day_rate = merchant_every_day_rate.groupby(['UID', 'merchant', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "merchant_every_day_rate = merchant_every_day_rate[['UID', 'trans_amt']]\n",
    "merchant_every_day_rate['trans_amt_before'] = merchant_every_day_rate.groupby('UID')['trans_amt'].shift(1)\n",
    "merchant_every_day_rate['trans_amt_day_rate'] = merchant_every_day_rate['trans_amt'] / merchant_every_day_rate['trans_amt_before']\n",
    "merchant_every_day_rate = merchant_every_day_rate[['UID', 'trans_amt_day_rate']]\n",
    "merchant_every_day_rate = merchant_every_day_rate.groupby('UID')['trans_amt_day_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, merchant_every_day_rate, on='UID', how='left')\n",
    "\n",
    "\n",
    "# 用户merchant每天半夜的统计量\n",
    "merchant_every_morning = transaction_data[transaction_data.hour <= 5][['UID', 'merchant', 'day', 'trans_amt']]\n",
    "merchant_every_morning = merchant_every_morning.groupby(['UID', 'merchant', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "merchant_every_morning = merchant_every_morning[['UID', 'trans_amt']]\n",
    "merchant_every_morning = merchant_every_morning.groupby('UID')['trans_amt'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "merchant_every_morning.columns = ['UID', 'merchant_every_morning_std', 'merchant_every_morning_mean', 'merchant_every_morning_min', 'merchant_every_morning_max']\n",
    "data = pd.merge(data, merchant_every_morning, on='UID', how='left')\n",
    "\n",
    "# 用户merchant每天半夜的gap\n",
    "merchant_every_morning_gap = transaction_data[transaction_data.hour <= 5][['UID', 'merchant', 'day', 'trans_amt']]\n",
    "merchant_every_morning_gap = merchant_every_morning_gap.groupby(['UID', 'merchant', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "merchant_every_morning_gap = merchant_every_morning_gap[['UID', 'trans_amt']]\n",
    "merchant_every_morning_gap['trans_amt_before'] = merchant_every_morning_gap.groupby('UID')['trans_amt'].shift(1)\n",
    "merchant_every_morning_gap['merchant_every_morning_gap'] = merchant_every_morning_gap['trans_amt'] - merchant_every_morning_gap['trans_amt_before']\n",
    "merchant_every_morning_gap = merchant_every_morning_gap[['UID', 'merchant_every_morning_gap']]\n",
    "merchant_every_morning_gap = merchant_every_morning_gap.groupby('UID')['merchant_every_morning_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "merchant_every_morning_gap.columns = ['UID', 'merchant_every_morning_gap_sum', 'merchant_every_morning_gap_std', 'merchant_every_morning_gap_mean', 'merchant_every_morning_gap_min', 'merchant_every_morning_gap_max']\n",
    "data = pd.merge(data, merchant_every_morning_gap, on='UID', how='left')\n",
    "\n",
    "# 用户merchant每天半夜的rate\n",
    "merchant_every_morning_rate = transaction_data[transaction_data.hour <= 5][['UID', 'merchant', 'day', 'trans_amt']]\n",
    "merchant_every_morning_rate = merchant_every_morning_rate.groupby(['UID', 'merchant', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "merchant_every_morning_rate = merchant_every_morning_rate[['UID', 'trans_amt']]\n",
    "merchant_every_morning_rate['trans_amt_before'] = merchant_every_morning_rate.groupby('UID')['trans_amt'].shift(1)\n",
    "merchant_every_morning_rate['merchant_every_morning_rate'] = merchant_every_morning_rate['trans_amt'] / merchant_every_morning_rate['trans_amt_before']\n",
    "merchant_every_morning_rate = merchant_every_morning_rate[['UID', 'merchant_every_morning_rate']]\n",
    "merchant_every_morning_rate = merchant_every_morning_rate.groupby('UID')['merchant_every_morning_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, merchant_every_morning_rate, on='UID', how='left')\n",
    "\n",
    "\n",
    "# 用户merchant每天早上的统计量\n",
    "merchant_every_work_time = transaction_data[(transaction_data.hour >= 6)&(transaction_data.hour <= 11)][['UID', 'merchant', 'day', 'trans_amt']]\n",
    "merchant_every_work_time = merchant_every_work_time.groupby(['UID', 'merchant', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "merchant_every_work_time = merchant_every_work_time[['UID', 'trans_amt']]\n",
    "merchant_every_work_time = merchant_every_work_time.groupby('UID')['trans_amt'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "merchant_every_work_time.columns = ['UID', 'merchant_every_work_time_std', 'merchant_every_work_time_mean', 'merchant_every_work_time_min', 'merchant_every_work_time_max']\n",
    "data = pd.merge(data, merchant_every_work_time, on='UID', how='left')\n",
    "\n",
    "# 用户merchant每天早上的gap\n",
    "merchant_every_work_time_gap = transaction_data[(transaction_data.hour >= 6)&(transaction_data.hour <= 11)][['UID', 'merchant', 'day', 'trans_amt']]\n",
    "merchant_every_work_time_gap = merchant_every_work_time_gap.groupby(['UID', 'merchant', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "merchant_every_work_time_gap = merchant_every_work_time_gap[['UID', 'trans_amt']]\n",
    "merchant_every_work_time_gap['trans_amt_before'] = merchant_every_work_time_gap.groupby('UID')['trans_amt'].shift(1)\n",
    "merchant_every_work_time_gap['merchant_every_work_time_gap'] = merchant_every_work_time_gap['trans_amt'] - merchant_every_work_time_gap['trans_amt_before']\n",
    "merchant_every_work_time_gap = merchant_every_work_time_gap[['UID', 'merchant_every_work_time_gap']]\n",
    "merchant_every_work_time_gap = merchant_every_work_time_gap.groupby('UID')['merchant_every_work_time_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "merchant_every_work_time_gap.columns = ['UID', 'merchant_every_work_time_gap_sum', 'merchant_every_work_time_gap_std', 'merchant_every_work_time_gap_mean', 'merchant_every_work_time_gap_min', 'merchant_every_work_time_gap_max']\n",
    "data = pd.merge(data, merchant_every_work_time_gap, on='UID', how='left')\n",
    "\n",
    "# 用户merchant每天早上的rate\n",
    "merchant_every_work_time_rate = transaction_data[(transaction_data.hour >= 6)&(transaction_data.hour <= 11)][['UID', 'merchant', 'day', 'trans_amt']]\n",
    "merchant_every_work_time_rate = merchant_every_work_time_rate.groupby(['UID', 'merchant', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "merchant_every_work_time_rate = merchant_every_work_time_rate[['UID', 'trans_amt']]\n",
    "merchant_every_work_time_rate['trans_amt_before'] = merchant_every_work_time_rate.groupby('UID')['trans_amt'].shift(1)\n",
    "merchant_every_work_time_rate['merchant_every_work_time_rate'] = merchant_every_work_time_rate['trans_amt'] / merchant_every_work_time_rate['trans_amt_before']\n",
    "merchant_every_work_time_rate = merchant_every_work_time_rate[['UID', 'merchant_every_work_time_rate']]\n",
    "merchant_every_work_time_rate = merchant_every_work_time_rate.groupby('UID')['merchant_every_work_time_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, merchant_every_work_time_rate, on='UID', how='left')\n",
    "\n",
    "# 用户merchant每天下午的统计量\n",
    "merchant_every_afternoon = transaction_data[(transaction_data.hour >= 12)&(transaction_data.hour <= 17)][['UID', 'merchant', 'day', 'trans_amt']]\n",
    "merchant_every_afternoon = merchant_every_afternoon.groupby(['UID', 'merchant', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "merchant_every_afternoon = merchant_every_afternoon[['UID', 'trans_amt']]\n",
    "merchant_every_afternoon = merchant_every_afternoon.groupby('UID')['trans_amt'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "merchant_every_afternoon.columns = ['UID', 'merchant_every_afternoon_std', 'merchant_every_afternoon_mean', 'merchant_every_afternoon_min', 'merchant_every_afternoon_max']\n",
    "data = pd.merge(data, merchant_every_afternoon, on='UID', how='left')\n",
    "\n",
    "# 用户merchant每天下午的gap\n",
    "merchant_every_afternoon_gap = transaction_data[(transaction_data.hour >= 12)&(transaction_data.hour <= 17)][['UID', 'merchant', 'day', 'trans_amt']]\n",
    "merchant_every_afternoon_gap = merchant_every_afternoon_gap.groupby(['UID', 'merchant', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "merchant_every_afternoon_gap = merchant_every_afternoon_gap[['UID', 'trans_amt']]\n",
    "merchant_every_afternoon_gap['trans_amt_before'] = merchant_every_afternoon_gap.groupby('UID')['trans_amt'].shift(1)\n",
    "merchant_every_afternoon_gap['merchant_every_afternoon_gap'] = merchant_every_afternoon_gap['trans_amt'] - merchant_every_afternoon_gap['trans_amt_before']\n",
    "merchant_every_afternoon_gap = merchant_every_afternoon_gap[['UID', 'merchant_every_afternoon_gap']]\n",
    "merchant_every_afternoon_gap = merchant_every_afternoon_gap.groupby('UID')['merchant_every_afternoon_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "merchant_every_afternoon_gap.columns = ['UID', 'merchant_every_afternoon_gap_sum', 'merchant_every_afternoon_gap_std', 'merchant_every_afternoon_gap_mean', 'merchant_every_afternoon_gap_min', 'merchant_every_afternoon_gap_max']\n",
    "data = pd.merge(data, merchant_every_afternoon_gap, on='UID', how='left')\n",
    "\n",
    "# 用户merchant每天下午的rate\n",
    "merchant_every_afternoon_rate = transaction_data[(transaction_data.hour >= 12)&(transaction_data.hour <= 17)][['UID', 'merchant', 'day', 'trans_amt']]\n",
    "merchant_every_afternoon_rate = merchant_every_afternoon_rate.groupby(['UID', 'merchant', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "merchant_every_afternoon_rate = merchant_every_afternoon_rate[['UID', 'trans_amt']]\n",
    "merchant_every_afternoon_rate['trans_amt_before'] = merchant_every_afternoon_rate.groupby('UID')['trans_amt'].shift(1)\n",
    "merchant_every_afternoon_rate['merchant_every_afternoon_rate'] = merchant_every_afternoon_rate['trans_amt'] / merchant_every_afternoon_rate['trans_amt_before']\n",
    "merchant_every_afternoon_rate = merchant_every_afternoon_rate[['UID', 'merchant_every_afternoon_rate']]\n",
    "merchant_every_afternoon_rate = merchant_every_afternoon_rate.groupby('UID')['merchant_every_afternoon_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, merchant_every_afternoon_rate, on='UID', how='left')\n",
    "\n",
    "# 用户merchant每天晚上的统计量\n",
    "merchant_every_night = transaction_data[(transaction_data.hour >= 18)&(transaction_data.hour <= 23)][['UID', 'merchant', 'day', 'trans_amt']]\n",
    "merchant_every_night = merchant_every_night.groupby(['UID', 'merchant', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "merchant_every_night = merchant_every_night[['UID', 'trans_amt']]\n",
    "merchant_every_night = merchant_every_night.groupby('UID')['trans_amt'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "merchant_every_night.columns = ['UID', 'merchant_every_night_std', 'merchant_every_night_mean', 'merchant_every_night_min', 'merchant_every_night_max']\n",
    "data = pd.merge(data, merchant_every_night, on='UID', how='left')\n",
    "\n",
    "# 用户merchant每天晚上的gap\n",
    "merchant_every_night_gap = transaction_data[(transaction_data.hour >= 18)&(transaction_data.hour <= 23)][['UID', 'merchant', 'day', 'trans_amt']]\n",
    "merchant_every_night_gap = merchant_every_night_gap.groupby(['UID', 'merchant', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "merchant_every_night_gap = merchant_every_night_gap[['UID', 'trans_amt']]\n",
    "merchant_every_night_gap['trans_amt_before'] = merchant_every_night_gap.groupby('UID')['trans_amt'].shift(1)\n",
    "merchant_every_night_gap['merchant_every_night_gap'] = merchant_every_night_gap['trans_amt'] - merchant_every_night_gap['trans_amt_before']\n",
    "merchant_every_night_gap = merchant_every_night_gap[['UID', 'merchant_every_night_gap']]\n",
    "merchant_every_night_gap = merchant_every_night_gap.groupby('UID')['merchant_every_night_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "merchant_every_night_gap.columns = ['UID', 'merchant_every_night_gap_sum', 'merchant_every_night_gap_std', 'merchant_every_night_gap_mean', 'merchant_every_night_gap_min', 'merchant_every_night_gap_max']\n",
    "data = pd.merge(data, merchant_every_night_gap, on='UID', how='left')\n",
    "\n",
    "# 用户merchant每天晚上的rate\n",
    "merchant_every_night_rate = transaction_data[(transaction_data.hour >= 18)&(transaction_data.hour <= 23)][['UID', 'merchant', 'day', 'trans_amt']]\n",
    "merchant_every_night_rate = merchant_every_night_rate.groupby(['UID', 'merchant', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "merchant_every_night_rate = merchant_every_night_rate[['UID', 'trans_amt']]\n",
    "merchant_every_night_rate['trans_amt_before'] = merchant_every_night_rate.groupby('UID')['trans_amt'].shift(1)\n",
    "merchant_every_night_rate['merchant_every_night_rate'] = merchant_every_night_rate['trans_amt'] / merchant_every_night_rate['trans_amt_before']\n",
    "merchant_every_night_rate = merchant_every_night_rate[['UID', 'merchant_every_night_rate']]\n",
    "merchant_every_night_rate = merchant_every_night_rate.groupby('UID')['merchant_every_night_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, merchant_every_night_rate, on='UID', how='left')\n",
    "\n",
    "\n",
    "# 用户merchant每三天的gap\n",
    "merchant_every_three_day_gap = transaction_data[['UID', 'merchant', 'day', 'trans_amt']]\n",
    "merchant_every_three_day_gap = merchant_every_three_day_gap.groupby(['UID', 'merchant', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "merchant_every_three_day_gap = merchant_every_three_day_gap[['UID', 'trans_amt']]\n",
    "merchant_every_three_day_gap['trans_amt_before'] = merchant_every_three_day_gap.groupby('UID')['trans_amt'].shift(3)\n",
    "merchant_every_three_day_gap['trans_amt_three_day_gap'] = merchant_every_three_day_gap['trans_amt'] - merchant_every_three_day_gap['trans_amt_before']\n",
    "merchant_every_three_day_gap = merchant_every_three_day_gap[['UID', 'trans_amt_three_day_gap']]\n",
    "merchant_every_three_day_gap = merchant_every_three_day_gap.groupby('UID')['trans_amt_three_day_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "merchant_every_three_day_gap.columns = ['UID', 'merchant_every_three_day_gap_sum', 'merchant_every_three_day_gap_std', 'merchant_every_three_day_gap_mean', 'merchant_every_three_day_gap_min', 'merchant_every_three_day_gap_max']\n",
    "data = pd.merge(data, merchant_every_three_day_gap, on='UID', how='left')\n",
    "\n",
    "# 用户merchant每三天的rate\n",
    "merchant_every_three_day_rate = transaction_data[['UID', 'merchant', 'day', 'trans_amt']]\n",
    "merchant_every_three_day_rate = merchant_every_three_day_rate.groupby(['UID', 'merchant', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "merchant_every_three_day_rate = merchant_every_three_day_rate[['UID', 'trans_amt']]\n",
    "merchant_every_three_day_rate['trans_amt_before'] = merchant_every_three_day_rate.groupby('UID')['trans_amt'].shift(3)\n",
    "merchant_every_three_day_rate['trans_amt_three_day_rate'] = merchant_every_three_day_rate['trans_amt'] / merchant_every_three_day_rate['trans_amt_before']\n",
    "merchant_every_three_day_rate = merchant_every_three_day_rate[['UID', 'trans_amt_three_day_rate']]\n",
    "merchant_every_three_day_rate = merchant_every_three_day_rate.groupby('UID')['trans_amt_three_day_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, merchant_every_three_day_rate, on='UID', how='left')\n",
    "\n",
    "\n",
    "# 用户merchant每七天的gap\n",
    "merchant_every_sixteen_day_gap = transaction_data[['UID', 'merchant', 'day', 'trans_amt']]\n",
    "merchant_every_sixteen_day_gap = merchant_every_sixteen_day_gap.groupby(['UID', 'merchant', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "merchant_every_sixteen_day_gap = merchant_every_sixteen_day_gap[['UID', 'trans_amt']]\n",
    "merchant_every_sixteen_day_gap['trans_amt_before'] = merchant_every_sixteen_day_gap.groupby('UID')['trans_amt'].shift(7)\n",
    "merchant_every_sixteen_day_gap['trans_amt_sixteen_day_gap'] = merchant_every_sixteen_day_gap['trans_amt'] - merchant_every_sixteen_day_gap['trans_amt_before']\n",
    "merchant_every_sixteen_day_gap = merchant_every_sixteen_day_gap[['UID', 'trans_amt_sixteen_day_gap']]\n",
    "merchant_every_sixteen_day_gap = merchant_every_sixteen_day_gap.groupby('UID')['trans_amt_sixteen_day_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "merchant_every_sixteen_day_gap.columns = ['UID', 'merchant_every_sixteen_day_gap_sum', 'merchant_every_sixteen_day_gap_std', 'merchant_every_sixteen_day_gap_mean', 'merchant_every_sixteen_day_gap_min', 'merchant_every_sixteen_day_gap_max']\n",
    "data = pd.merge(data, merchant_every_sixteen_day_gap, on='UID', how='left')\n",
    "\n",
    "# 用户merchant每七天的rate\n",
    "merchant_every_sixteen_day_rate = transaction_data[['UID', 'merchant', 'day', 'trans_amt']]\n",
    "merchant_every_sixteen_day_rate = merchant_every_sixteen_day_rate.groupby(['UID', 'merchant', 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "merchant_every_sixteen_day_rate = merchant_every_sixteen_day_rate[['UID', 'trans_amt']]\n",
    "merchant_every_sixteen_day_rate['trans_amt_before'] = merchant_every_sixteen_day_rate.groupby('UID')['trans_amt'].shift(7)\n",
    "merchant_every_sixteen_day_rate['trans_amt_sixteen_day_rate'] = merchant_every_sixteen_day_rate['trans_amt'] / merchant_every_sixteen_day_rate['trans_amt_before']\n",
    "merchant_every_sixteen_day_rate = merchant_every_sixteen_day_rate[['UID', 'trans_amt_sixteen_day_rate']]\n",
    "merchant_every_sixteen_day_rate = merchant_every_sixteen_day_rate.groupby('UID')['trans_amt_sixteen_day_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, merchant_every_sixteen_day_rate, on='UID', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mac1 of operation_data is over!\n",
      "mac2 of operation_data is over!\n",
      "ip1_sub of operation_data is over!\n",
      "ip2_sub of operation_data is over!\n",
      "merchant of transaction_data is over!\n",
      "mac1 of transaction_data is over!\n",
      "ip1_sub of transaction_data is over!\n",
      "market_code of transaction_data is over!\n"
     ]
    }
   ],
   "source": [
    "def frequence_device_time(data, temp, x, t):\n",
    "    \"\"\"\n",
    "    label和时间交叉count\n",
    "    \"\"\"\n",
    "    # 每秒设备的count\n",
    "    second = temp[[x, 'day', 'time']]\n",
    "    second[x + '_second_count'] = 1\n",
    "    second = second.groupby([x, 'day', 'time']).agg('count').reset_index()\n",
    "    second = pd.merge(temp[['UID', x, 'day', 'time']], second, on=[x, 'day', 'time'], how='left')\n",
    "    second = second[['UID', x + '_second_count']]\n",
    "    second = second.groupby('UID')[x + '_second_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    second.columns = ['UID', x + '_second_std', x + '_second_mean', x + '_second_min', x + '_second_max']\n",
    "    data = pd.merge(data, second, on='UID', how='left')\n",
    "    \n",
    "    # 每分钟设备的count\n",
    "    temp['minute'] = temp['time'].apply(lambda x: str(int(x.split(':')[0])) + str(int(x.split(':')[1])))\n",
    "    minute = temp[[x, 'day', 'minute']]\n",
    "    minute[x + '_minute_count'] = 1\n",
    "    minute = minute.groupby([x, 'day', 'minute']).agg('count').reset_index()\n",
    "    minute = pd.merge(temp[['UID', x, 'day', 'minute']], minute, on=[x, 'day', 'minute'], how='left')\n",
    "    minute = minute[['UID', x + '_minute_count']]\n",
    "    minute = minute.groupby('UID')[x + '_minute_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    minute.columns = ['UID', x + '_minute_std', x + '_minute_mean', x + '_minute_min', x + '_minute_max']\n",
    "    data = pd.merge(data, minute, on='UID', how='left')\n",
    "    \n",
    "    # 每一天的每一小时的count\n",
    "    hour = temp[[x, 'day', 'hour']]\n",
    "    hour[x + '_hour_count'] = 1\n",
    "    hour = hour.groupby([x, 'day', 'hour']).agg('count').reset_index()\n",
    "    hour = pd.merge(temp[['UID', x, 'day', 'hour']], hour, on=[x, 'day', 'hour'], how='left')\n",
    "    hour = hour[['UID', x + '_hour_count']]\n",
    "    hour = hour.groupby('UID')[x + '_hour_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    hour.columns = ['UID', x + '_hour_std', x + '_hour_mean', x + '_hour_min', x + '_hour_max']\n",
    "    data = pd.merge(data, hour, on='UID', how='left')\n",
    "    \n",
    "    # 每一天的每一小时的count_gap\n",
    "    hour_gap = temp[[x, 'day', 'hour']]\n",
    "    hour_gap[x + '_hour_gap_count'] = 1\n",
    "    hour_gap = hour_gap.groupby([x, 'day', 'hour']).agg('count').reset_index()\n",
    "    hour_gap['before_hour_count'] = hour_gap.groupby(x)[x + '_hour_gap_count'].shift(1)\n",
    "    hour_gap[x + 'hour_count_gap'] = hour_gap[x + '_hour_gap_count'] - hour_gap['before_hour_count']\n",
    "    hour_gap = hour_gap.drop([x + '_hour_gap_count', 'before_hour_count'], axis=1)\n",
    "    hour_gap = pd.merge(temp[['UID', x, 'day', 'hour']], hour_gap, on=[x, 'day', 'hour'], how='left')\n",
    "    hour_gap = hour_gap[['UID', x + 'hour_count_gap']]\n",
    "    hour_gap = hour_gap.groupby('UID')[x + 'hour_count_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    hour_gap.columns = ['UID', x + '_hour_gap_sum', x + '_hour_gap_std', x + '_hour_gap_mean', x + '_hour_gap_min', x + '_hour_gap_max']\n",
    "    data = pd.merge(data, hour_gap, on='UID', how='left')\n",
    "    \n",
    "    # 每一天的每一小时的count_rate\n",
    "    hour_rate = temp[[x, 'day', 'hour']]\n",
    "    hour_rate[x + '_hour_rate_count'] = 1\n",
    "    hour_rate = hour_rate.groupby([x, 'day', 'hour']).agg('count').reset_index()\n",
    "    hour_rate['before_hour_count'] = hour_rate.groupby(x)[x + '_hour_rate_count'].shift(1)\n",
    "    hour_rate[x + 'hour_count_rate'] = hour_rate[x + '_hour_rate_count'] / hour_rate['before_hour_count']\n",
    "    hour_rate = hour_rate.drop([x + '_hour_rate_count', 'before_hour_count'], axis=1)\n",
    "    hour_rate = pd.merge(temp[['UID', x, 'day', 'hour']], hour_rate, on=[x, 'day', 'hour'], how='left')\n",
    "    hour_rate = hour_rate[['UID', x + 'hour_count_rate']]\n",
    "    hour_rate = hour_rate.groupby('UID')[x + 'hour_count_rate'].agg('sum').reset_index()\n",
    "    data = pd.merge(data, hour_rate, on='UID', how='left')\n",
    "    \n",
    "    # 每一天的count\n",
    "    day = temp[[x, 'day']]\n",
    "    day[x + '_day_count'] = 1\n",
    "    day = day.groupby([x, 'day']).agg('count').reset_index()\n",
    "    day = pd.merge(temp[['UID', x, 'day']], day, on=[x, 'day'], how='left')\n",
    "    day = day[['UID', x + '_day_count']]\n",
    "    day = day.groupby('UID')[x + '_day_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    day.columns = ['UID', x + '_day_std', x + '_day_mean', x + '_day_min', x + '_day_max']\n",
    "    data = pd.merge(data, day, on='UID', how='left')\n",
    "    \n",
    "    # 每一天的count_gap\n",
    "    day_gap = temp[[x, 'day']]\n",
    "    day_gap[x + '_day_gap_count'] = 1\n",
    "    day_gap = day_gap.groupby([x, 'day']).agg('count').reset_index()\n",
    "    day_gap['before_day_count'] = day_gap.groupby(x)[x + '_day_gap_count'].shift(1)\n",
    "    day_gap[x + 'day_count_gap'] = day_gap[x + '_day_gap_count'] - day_gap['before_day_count']\n",
    "    day_gap = day_gap.drop([x + '_day_gap_count', 'before_day_count'], axis=1)\n",
    "    day_gap = pd.merge(temp[['UID', x, 'day']], day_gap, on=[x, 'day'], how='left')\n",
    "    day_gap = day_gap[['UID', x + 'day_count_gap']]\n",
    "    day_gap = day_gap.groupby('UID')[x + 'day_count_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    day_gap.columns = ['UID', x + '_day_gap_sum', x + '_day_gap_std', x + '_day_gap_mean', x + '_day_gap_min', x + '_day_gap_max']\n",
    "    data = pd.merge(data, day_gap, on='UID', how='left')\n",
    "    \n",
    "    # 每一天的count_rate\n",
    "    day_rate = temp[[x, 'day']]\n",
    "    day_rate[x + '_day_rate_count'] = 1\n",
    "    day_rate = day_rate.groupby([x, 'day']).agg('count').reset_index()\n",
    "    day_rate['before_day_count'] = day_rate.groupby(x)[x + '_day_rate_count'].shift(1)\n",
    "    day_rate[x + 'day_count_rate'] = day_rate[x + '_day_rate_count'] / day_rate['before_day_count']\n",
    "    day_rate = day_rate.drop([x + '_day_rate_count', 'before_day_count'], axis=1)\n",
    "    day_rate = pd.merge(temp[['UID', x, 'day']], day_rate, on=[x, 'day'], how='left')\n",
    "    day_rate = day_rate[['UID', x + 'day_count_rate']]\n",
    "    day_rate = day_rate.groupby('UID')[x + 'day_count_rate'].agg('sum').reset_index()\n",
    "    data = pd.merge(data, day_rate, on='UID', how='left')\n",
    "    \n",
    "    # 每一天半夜的count\n",
    "    morning = temp[temp.hour <= 5][[x, 'day']]\n",
    "    morning[x + '_morning_count'] = 1\n",
    "    morning = morning.groupby([x, 'day']).agg('count').reset_index()\n",
    "    morning = pd.merge(temp[['UID', x, 'day']], morning, on=[x, 'day'], how='left')\n",
    "    morning = morning[['UID', x + '_morning_count']]\n",
    "    morning = morning.groupby('UID')[x + '_morning_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    morning.columns = ['UID', x + '_morning_std', x + '_morning_mean', x + '_morning_min', x + '_morning_max']\n",
    "    data = pd.merge(data, morning, on='UID', how='left')\n",
    "    \n",
    "    # 每一天早上的count\n",
    "    work_time = temp[(temp.hour >= 6)&(temp.hour <= 11)][[x, 'day']]\n",
    "    work_time[x + '_work_time_count'] = 1\n",
    "    work_time = work_time.groupby([x, 'day']).agg('count').reset_index()\n",
    "    work_time = pd.merge(temp[['UID', x, 'day']], work_time, on=[x, 'day'], how='left')\n",
    "    work_time = work_time[['UID', x + '_work_time_count']]\n",
    "    work_time = work_time.groupby('UID')[x + '_work_time_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    work_time.columns = ['UID', x + '_work_time_std', x + '_work_time_mean', x + '_work_time_min', x + '_work_time_max']\n",
    "    data = pd.merge(data, work_time, on='UID', how='left')\n",
    "    \n",
    "    # 每一天下午的count\n",
    "    afternoon = temp[(temp.hour >= 12)&(temp.hour <= 17)][[x, 'day']]\n",
    "    afternoon[x + '_afternoon_count'] = 1\n",
    "    afternoon = afternoon.groupby([x, 'day']).agg('count').reset_index()\n",
    "    afternoon = pd.merge(temp[['UID', x, 'day']], afternoon, on=[x, 'day'], how='left')\n",
    "    afternoon = afternoon[['UID', x + '_afternoon_count']]\n",
    "    afternoon = afternoon.groupby('UID')[x + '_afternoon_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    afternoon.columns = ['UID', x + '_afternoon_std', x + '_afternoon_mean', x + '_afternoon_min', x + '_afternoon_max']\n",
    "    data = pd.merge(data, afternoon, on='UID', how='left')\n",
    "    \n",
    "    # 每一天晚上的count\n",
    "    night = temp[(temp.hour >= 18)&(temp.hour <= 23)][[x, 'day']]\n",
    "    night[x + '_night_count'] = 1\n",
    "    night = night.groupby([x, 'day']).agg('count').reset_index()\n",
    "    night = pd.merge(temp[['UID', x, 'day']], night, on=[x, 'day'], how='left')\n",
    "    night = night[['UID', x + '_night_count']]\n",
    "    night = night.groupby('UID')[x + '_night_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    night.columns = ['UID', x + '_night_std', x + '_night_mean', x + '_night_min', x + '_night_max']\n",
    "    data = pd.merge(data, night, on='UID', how='left')\n",
    "    \n",
    "    # 每一天半夜的gap\n",
    "    morning_gap = temp[temp.hour <= 5][[x, 'day']]\n",
    "    morning_gap[x + '_morning_gap_count'] = 1\n",
    "    morning_gap = morning_gap.groupby([x, 'day']).agg('count').reset_index()\n",
    "    morning_gap = pd.merge(temp[['UID', x, 'day']], morning_gap, on=[x, 'day'], how='left')\n",
    "    morning_gap[x + '_morning_gap_count_before'] = morning_gap.groupby('UID')[x + '_morning_gap_count'].shift(1)\n",
    "    morning_gap[x + '_morning_gap_count_gap'] = morning_gap[x + '_morning_gap_count'] - morning_gap[x + '_morning_gap_count_before']\n",
    "    morning_gap = morning_gap[['UID', x + '_morning_gap_count_gap']]\n",
    "    morning_gap = morning_gap.groupby('UID')[x + '_morning_gap_count_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    morning_gap.columns = ['UID', x + '_morning_gap_sum', x + '_morning_gap_std', x + '_morning_gap_mean', x + '_morning_gap_min', x + '_morning_gap_max']\n",
    "    data = pd.merge(data, morning_gap, on='UID', how='left')\n",
    "    \n",
    "    # 每一天早上的gap\n",
    "    work_time_gap = temp[(temp.hour >= 6)&(temp.hour <= 11)][[x, 'day']]\n",
    "    work_time_gap[x + '_work_time_gap_count'] = 1\n",
    "    work_time_gap = work_time_gap.groupby([x, 'day']).agg('count').reset_index()\n",
    "    work_time_gap = pd.merge(temp[['UID', x, 'day']], work_time_gap, on=[x, 'day'], how='left')\n",
    "    work_time_gap[x + '_work_time_gap_count_before'] = work_time_gap.groupby('UID')[x + '_work_time_gap_count'].shift(1)\n",
    "    work_time_gap[x + '_work_time_gap_count_gap'] = work_time_gap[x + '_work_time_gap_count'] - work_time_gap[x + '_work_time_gap_count_before']\n",
    "    work_time_gap = work_time_gap[['UID', x + '_work_time_gap_count_gap']]\n",
    "    work_time_gap = work_time_gap.groupby('UID')[x + '_work_time_gap_count_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    work_time_gap.columns = ['UID', x + '_work_time_gap_sum', x + '_work_time_gap_std', x + '_work_time_gap_mean', x + '_work_time_gap_min', x + '_work_time_gap_max']\n",
    "    data = pd.merge(data, work_time_gap, on='UID', how='left')\n",
    "    \n",
    "    # 每一天下午的gap\n",
    "    afternoon_gap = temp[(temp.hour >= 12)&(temp.hour <= 17)][[x, 'day']]\n",
    "    afternoon_gap[x + '_afternoon_gap_count'] = 1\n",
    "    afternoon_gap = afternoon_gap.groupby([x, 'day']).agg('count').reset_index()\n",
    "    afternoon_gap = pd.merge(temp[['UID', x, 'day']], afternoon_gap, on=[x, 'day'], how='left')\n",
    "    afternoon_gap[x + '_afternoon_gap_count_before'] = afternoon_gap.groupby('UID')[x + '_afternoon_gap_count'].shift(1)\n",
    "    afternoon_gap[x + '_afternoon_gap_count_gap'] = afternoon_gap[x + '_afternoon_gap_count'] - afternoon_gap[x + '_afternoon_gap_count_before']\n",
    "    afternoon_gap = afternoon_gap[['UID', x + '_afternoon_gap_count_gap']]\n",
    "    afternoon_gap = afternoon_gap.groupby('UID')[x + '_afternoon_gap_count_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    afternoon_gap.columns = ['UID', x + '_afternoon_gap_sum', x + '_afternoon_gap_std', x + '_afternoon_gap_mean', x + '_afternoon_gap_min', x + '_afternoon_gap_max']\n",
    "    data = pd.merge(data, afternoon_gap, on='UID', how='left')\n",
    "    \n",
    "    # 每一天晚上的gap\n",
    "    night_gap = temp[(temp.hour >= 18)&(temp.hour <= 23)][[x, 'day']]\n",
    "    night_gap[x + '_night_gap_count'] = 1\n",
    "    night_gap = night_gap.groupby([x, 'day']).agg('count').reset_index()\n",
    "    night_gap = pd.merge(temp[['UID', x, 'day']], night_gap, on=[x, 'day'], how='left')\n",
    "    night_gap[x + '_night_gap_count_before'] = night_gap.groupby('UID')[x + '_night_gap_count'].shift(1)\n",
    "    night_gap[x + '_night_gap_count_gap'] = night_gap[x + '_night_gap_count'] - night_gap[x + '_night_gap_count_before']\n",
    "    night_gap = night_gap[['UID', x + '_night_gap_count_gap']]\n",
    "    night_gap = night_gap.groupby('UID')[x + '_night_gap_count_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    night_gap.columns = ['UID', x + '_night_gap_sum', x + '_night_gap_std', x + '_night_gap_mean', x + '_night_gap_min', x + '_night_gap_max']\n",
    "    data = pd.merge(data, night_gap, on='UID', how='left')\n",
    "    \n",
    "    \n",
    "    # 每一天半夜的rate\n",
    "    morning_rate = temp[temp.hour <= 5][[x, 'day']]\n",
    "    morning_rate[x + '_morning_rate_count'] = 1\n",
    "    morning_rate = morning_rate.groupby([x, 'day']).agg('count').reset_index()\n",
    "    morning_rate = pd.merge(temp[['UID', x, 'day']], morning_rate, on=[x, 'day'], how='left')\n",
    "    morning_rate[x + '_morning_rate_count_before'] = morning_rate.groupby('UID')[x + '_morning_rate_count'].shift(1)\n",
    "    morning_rate[x + '_morning_rate_count_rate'] = morning_rate[x + '_morning_rate_count'] / morning_rate[x + '_morning_rate_count_before']\n",
    "    morning_rate = morning_rate[['UID', x + '_morning_rate_count_rate']]\n",
    "    morning_rate = morning_rate.groupby('UID')[x + '_morning_rate_count_rate'].agg('sum').reset_index()\n",
    "    data = pd.merge(data, morning_rate, on='UID', how='left')\n",
    "    \n",
    "    # 每一天早上的rate\n",
    "    work_time_rate = temp[(temp.hour >= 6)&(temp.hour <= 11)][[x, 'day']]\n",
    "    work_time_rate[x + '_work_time_rate_count'] = 1\n",
    "    work_time_rate = work_time_rate.groupby([x, 'day']).agg('count').reset_index()\n",
    "    work_time_rate = pd.merge(temp[['UID', x, 'day']], work_time_rate, on=[x, 'day'], how='left')\n",
    "    work_time_rate[x + '_work_time_rate_count_before'] = work_time_rate.groupby('UID')[x + '_work_time_rate_count'].shift(1)\n",
    "    work_time_rate[x + '_work_time_rate_count_rate'] = work_time_rate[x + '_work_time_rate_count'] / work_time_rate[x + '_work_time_rate_count_before']\n",
    "    work_time_rate = work_time_rate[['UID', x + '_work_time_rate_count_rate']]\n",
    "    work_time_rate = work_time_rate.groupby('UID')[x + '_work_time_rate_count_rate'].agg('sum').reset_index()\n",
    "    data = pd.merge(data, work_time_rate, on='UID', how='left')\n",
    "    \n",
    "    # 每一天下午的rate\n",
    "    afternoon_rate = temp[(temp.hour >= 12)&(temp.hour <= 17)][[x, 'day']]\n",
    "    afternoon_rate[x + '_afternoon_rate_count'] = 1\n",
    "    afternoon_rate = afternoon_rate.groupby([x, 'day']).agg('count').reset_index()\n",
    "    afternoon_rate = pd.merge(temp[['UID', x, 'day']], afternoon_rate, on=[x, 'day'], how='left')\n",
    "    afternoon_rate[x + '_afternoon_rate_count_before'] = afternoon_rate.groupby('UID')[x + '_afternoon_rate_count'].shift(1)\n",
    "    afternoon_rate[x + '_afternoon_rate_count_rate'] = afternoon_rate[x + '_afternoon_rate_count'] / afternoon_rate[x + '_afternoon_rate_count_before']\n",
    "    afternoon_rate = afternoon_rate[['UID', x + '_afternoon_rate_count_rate']]\n",
    "    afternoon_rate = afternoon_rate.groupby('UID')[x + '_afternoon_rate_count_rate'].agg('sum').reset_index()\n",
    "    data = pd.merge(data, afternoon_rate, on='UID', how='left')\n",
    "    \n",
    "    # 每一天晚上的rate\n",
    "    night_rate = temp[(temp.hour >= 18)&(temp.hour <= 23)][[x, 'day']]\n",
    "    night_rate[x + '_night_rate_count'] = 1\n",
    "    night_rate = night_rate.groupby([x, 'day']).agg('count').reset_index()\n",
    "    night_rate = pd.merge(temp[['UID', x, 'day']], night_rate, on=[x, 'day'], how='left')\n",
    "    night_rate[x + '_night_rate_count_before'] = night_rate.groupby('UID')[x + '_night_rate_count'].shift(1)\n",
    "    night_rate[x + '_night_rate_count_rate'] = night_rate[x + '_night_rate_count'] / night_rate[x + '_night_rate_count_before']\n",
    "    night_rate = night_rate[['UID', x + '_night_rate_count_rate']]\n",
    "    night_rate = night_rate.groupby('UID')[x + '_night_rate_count_rate'].agg('sum').reset_index()\n",
    "    data = pd.merge(data, night_rate, on='UID', how='left')\n",
    "    \n",
    "    \n",
    "    # 每三天的gap\n",
    "    three_day_gap = temp[[x, 'day']]\n",
    "    three_day_gap[x + '_three_day_gap_count'] = 1\n",
    "    three_day_gap = three_day_gap.groupby([x, 'day']).agg('count').reset_index()\n",
    "    three_day_gap['before_three_day_count'] = three_day_gap.groupby(x)[x + '_three_day_gap_count'].shift(3)\n",
    "    three_day_gap[x + 'three_day_count_gap'] = three_day_gap[x + '_three_day_gap_count'] - three_day_gap['before_three_day_count']\n",
    "    three_day_gap = three_day_gap.drop([x + '_three_day_gap_count', 'before_three_day_count'], axis=1)\n",
    "    three_day_gap = pd.merge(temp[['UID', x, 'day']], three_day_gap, on=[x, 'day'], how='left')\n",
    "    three_day_gap = three_day_gap[['UID', x + 'three_day_count_gap']]\n",
    "    three_day_gap = three_day_gap.groupby('UID')[x + 'three_day_count_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    three_day_gap.columns = ['UID', x + '_three_day_gap_sum', x + '_three_day_gap_std', x + '_three_day_gap_mean', x + '_three_day_gap_min', x + '_three_day_gap_max']\n",
    "    data = pd.merge(data, three_day_gap, on='UID', how='left')\n",
    "    \n",
    "    # 每三天的count_rate\n",
    "    three_day_rate = temp[[x, 'day']]\n",
    "    three_day_rate[x + '_three_day_rate_count'] = 1\n",
    "    three_day_rate = three_day_rate.groupby([x, 'day']).agg('count').reset_index()\n",
    "    three_day_rate['before_three_day_count'] = three_day_rate.groupby(x)[x + '_three_day_rate_count'].shift(3)\n",
    "    three_day_rate[x + 'three_day_count_rate'] = three_day_rate[x + '_three_day_rate_count'] / three_day_rate['before_three_day_count']\n",
    "    three_day_rate = three_day_rate.drop([x + '_three_day_rate_count', 'before_three_day_count'], axis=1)\n",
    "    three_day_rate = pd.merge(temp[['UID', x, 'day']], three_day_rate, on=[x, 'day'], how='left')\n",
    "    three_day_rate = three_day_rate[['UID', x + 'three_day_count_rate']]\n",
    "    three_day_rate = three_day_rate.groupby('UID')[x + 'three_day_count_rate'].agg('sum').reset_index()\n",
    "    data = pd.merge(data, three_day_rate, on='UID', how='left')\n",
    "    \n",
    "    \n",
    "    # 每七天的gap\n",
    "    sixteen_day_gap = temp[[x, 'day']]\n",
    "    sixteen_day_gap[x + '_sixteen_day_gap_count'] = 1\n",
    "    sixteen_day_gap = sixteen_day_gap.groupby([x, 'day']).agg('count').reset_index()\n",
    "    sixteen_day_gap['before_sixteen_day_count'] = sixteen_day_gap.groupby(x)[x + '_sixteen_day_gap_count'].shift(7)\n",
    "    sixteen_day_gap[x + 'sixteen_day_count_gap'] = sixteen_day_gap[x + '_sixteen_day_gap_count'] - sixteen_day_gap['before_sixteen_day_count']\n",
    "    sixteen_day_gap = sixteen_day_gap.drop([x + '_sixteen_day_gap_count', 'before_sixteen_day_count'], axis=1)\n",
    "    sixteen_day_gap = pd.merge(temp[['UID', x, 'day']], sixteen_day_gap, on=[x, 'day'], how='left')\n",
    "    sixteen_day_gap = sixteen_day_gap[['UID', x + 'sixteen_day_count_gap']]\n",
    "    sixteen_day_gap = sixteen_day_gap.groupby('UID')[x + 'sixteen_day_count_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    sixteen_day_gap.columns = ['UID', x + '_sixteen_day_gap_sum', x + '_sixteen_day_gap_std', x + '_sixteen_day_gap_mean', x + '_sixteen_day_gap_min', x + '_sixteen_day_gap_max']\n",
    "    data = pd.merge(data, sixteen_day_gap, on='UID', how='left')\n",
    "    \n",
    "    # 每七天的count_rate\n",
    "    sixteen_day_rate = temp[[x, 'day']]\n",
    "    sixteen_day_rate[x + '_sixteen_day_rate_count'] = 1\n",
    "    sixteen_day_rate = sixteen_day_rate.groupby([x, 'day']).agg('count').reset_index()\n",
    "    sixteen_day_rate['before_sixteen_day_count'] = sixteen_day_rate.groupby(x)[x + '_sixteen_day_rate_count'].shift(7)\n",
    "    sixteen_day_rate[x + '_sixteen_day_count_rate'] = sixteen_day_rate[x + '_sixteen_day_rate_count'] / sixteen_day_rate['before_sixteen_day_count']\n",
    "    sixteen_day_rate = sixteen_day_rate.drop([x + '_sixteen_day_rate_count', 'before_sixteen_day_count'], axis=1)\n",
    "    sixteen_day_rate = pd.merge(temp[['UID', x, 'day']], sixteen_day_rate, on=[x, 'day'], how='left')\n",
    "    sixteen_day_rate = sixteen_day_rate[['UID', x + '_sixteen_day_count_rate']]\n",
    "    sixteen_day_rate = sixteen_day_rate.groupby('UID')[x + '_sixteen_day_count_rate'].agg('sum').reset_index()\n",
    "    data = pd.merge(data, sixteen_day_rate, on='UID', how='left')\n",
    "    \n",
    "    print(x + ' of ' + t, 'is over!')\n",
    "    return data\n",
    "\n",
    "#data = frequence_device_time(data, transaction_data, 'merchant', 'transaction_data')\n",
    "# 原来是mac和ip_sub\n",
    "for x in ['mac1', 'mac2', 'ip1_sub', 'ip2_sub']:\n",
    "    data = frequence_device_time(data, operation_data, x, 'operation_data')\n",
    "\n",
    "for y in ['merchant', 'mac1', 'ip1_sub', 'market_code']:\n",
    "    data = frequence_device_time(data, transaction_data, y, 'transaction_data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "merchant is over!\n",
      "mac1 is over!\n",
      "ip1_sub is over!\n",
      "market_code is over!\n"
     ]
    }
   ],
   "source": [
    "def amt_label_frequence(data, temp, x):\n",
    "    # 每天的每一小时用该x(如ip)进行交易的钱数\n",
    "    every_day_hour_amt = temp[[x, 'day', 'hour', 'trans_amt']]\n",
    "    every_day_hour_amt['every_day_' + x + '_amt'] = every_day_hour_amt['trans_amt']\n",
    "    every_day_hour_amt = every_day_hour_amt.groupby([x, 'day', 'hour'])['every_day_' + x + '_amt'].agg('sum').reset_index()\n",
    "    every_day_hour_amt = pd.merge(temp[['UID', x, 'day', 'hour']], every_day_hour_amt, on=[x, 'day', 'hour'], how='left')\n",
    "    every_day_hour_amt = every_day_hour_amt[['UID', 'every_day_' + x + '_amt']]\n",
    "    every_day_hour_amt = every_day_hour_amt.groupby('UID')['every_day_' + x + '_amt'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    every_day_hour_amt.columns = ['UID', 'every_day_hour_amt_std_' + x, 'every_day_hour_amt_mean_' + x, 'every_day_hour_amt_min_' + x, 'every_day_hour_amt_max_' + x]\n",
    "    data = pd.merge(data, every_day_hour_amt, on='UID', how='left')\n",
    "    \n",
    "    # 每天的每一小时用该x(如ip)进行交易的钱数的gap\n",
    "    every_day_hour_amt_gap = temp[[x, 'day', 'hour', 'trans_amt']]\n",
    "    every_day_hour_amt_gap['every_day_' + x + '_amt'] = every_day_hour_amt_gap['trans_amt']\n",
    "    every_day_hour_amt_gap = every_day_hour_amt_gap.groupby([x, 'day', 'hour'])['every_day_' + x + '_amt'].agg('sum').reset_index()\n",
    "    every_day_hour_amt_gap = pd.merge(temp[['UID', x, 'day', 'hour']], every_day_hour_amt_gap, on=[x, 'day', 'hour'], how='left')\n",
    "    every_day_hour_amt_gap = every_day_hour_amt_gap[['UID', 'every_day_' + x + '_amt']]\n",
    "    every_day_hour_amt_gap['before_every_day_hour_amt'] = every_day_hour_amt_gap.groupby('UID')['every_day_' + x + '_amt'].shift(1)\n",
    "    every_day_hour_amt_gap['every_day_hour_amt_gap_' + 'of_' + x] = every_day_hour_amt_gap['every_day_' + x + '_amt'] - every_day_hour_amt_gap['before_every_day_hour_amt']\n",
    "    every_day_hour_amt_gap = every_day_hour_amt_gap[['UID', 'every_day_hour_amt_gap_' + 'of_' + x]]\n",
    "    every_day_hour_amt_gap = every_day_hour_amt_gap.groupby('UID')['every_day_hour_amt_gap_' + 'of_' + x].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    every_day_hour_amt_gap.columns = ['UID', 'every_day_hour_amt_gap_sum_' + x, 'every_day_hour_amt_gap_std_' + x, 'every_day_hour_amt_gap_mean_' + x, 'every_day_hour_amt_gap_min_' + x, 'every_day_hour_amt_gap_max_' + x]\n",
    "    data = pd.merge(data, every_day_hour_amt_gap, on='UID', how='left')\n",
    "    \n",
    "     # 每天的每一小时用该x(如ip)进行交易的钱数的rate\n",
    "    every_day_hour_amt_rate = temp[[x, 'day', 'hour', 'trans_amt']]\n",
    "    every_day_hour_amt_rate['every_day_' + x + '_amt'] = every_day_hour_amt_rate['trans_amt']\n",
    "    every_day_hour_amt_rate = every_day_hour_amt_rate.groupby([x, 'day', 'hour'])['every_day_' + x + '_amt'].agg('sum').reset_index()\n",
    "    every_day_hour_amt_rate = pd.merge(temp[['UID', x, 'day', 'hour']], every_day_hour_amt_rate, on=[x, 'day', 'hour'], how='left')\n",
    "    every_day_hour_amt_rate = every_day_hour_amt_rate[['UID', 'every_day_' + x + '_amt']]\n",
    "    every_day_hour_amt_rate['before_every_day_hour_amt'] = every_day_hour_amt_rate.groupby('UID')['every_day_' + x + '_amt'].shift(1)\n",
    "    every_day_hour_amt_rate['every_day_hour_amt_rate_' + 'of_' + x] = every_day_hour_amt_rate['every_day_' + x + '_amt'] / every_day_hour_amt_rate['before_every_day_hour_amt']\n",
    "    every_day_hour_amt_rate = every_day_hour_amt_rate[['UID', 'every_day_hour_amt_rate_' + 'of_' + x]]\n",
    "    every_day_hour_amt_rate = every_day_hour_amt_rate.groupby('UID')['every_day_hour_amt_rate_' + 'of_' + x].agg('sum').reset_index()\n",
    "    data = pd.merge(data, every_day_hour_amt_rate, on='UID', how='left')\n",
    "    \n",
    "    \n",
    "    \n",
    "    # 每天用该x(如ip)进行交易的钱数\n",
    "    every_day_amt = temp[[x, 'day', 'trans_amt']]\n",
    "    every_day_amt['every_day_' + x + '_amt'] = every_day_amt['trans_amt']\n",
    "    every_day_amt = every_day_amt.groupby([x, 'day'])['every_day_' + x + '_amt'].agg('sum').reset_index()\n",
    "    every_day_amt = pd.merge(temp[['UID', x, 'day']], every_day_amt, on=[x, 'day'], how='left')\n",
    "    every_day_amt = every_day_amt[['UID', 'every_day_' + x + '_amt']]\n",
    "    every_day_amt = every_day_amt.groupby('UID')['every_day_' + x + '_amt'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    every_day_amt.columns = ['UID', 'every_day_amt_std_' + x, 'every_day_amt_mean_' + x, 'every_day_amt_min_' + x, 'every_day_amt_max_' + x]\n",
    "    data = pd.merge(data, every_day_amt, on='UID', how='left')\n",
    "\n",
    "    # 每天用该x(如ip)进行交易的钱数的gap\n",
    "    every_day_amt_gap = temp[[x, 'day', 'trans_amt']]\n",
    "    every_day_amt_gap['every_day_' + x + '_amt'] = every_day_amt_gap['trans_amt']\n",
    "    every_day_amt_gap = every_day_amt_gap.groupby([x, 'day'])['every_day_' + x + '_amt'].agg('sum').reset_index()\n",
    "    every_day_amt_gap = pd.merge(temp[['UID', x, 'day']], every_day_amt_gap, on=[x, 'day'], how='left')\n",
    "    every_day_amt_gap = every_day_amt_gap[['UID', 'every_day_' + x + '_amt']]\n",
    "    every_day_amt_gap['before_every_day_amt'] = every_day_amt_gap.groupby('UID')['every_day_' + x + '_amt'].shift(1)\n",
    "    every_day_amt_gap['every_day_amt_gap_' + 'of_' + x] = every_day_amt_gap['every_day_' + x + '_amt'] - every_day_amt_gap['before_every_day_amt']\n",
    "    every_day_amt_gap = every_day_amt_gap[['UID', 'every_day_amt_gap_' + 'of_' + x]]\n",
    "    every_day_amt_gap = every_day_amt_gap.groupby('UID')['every_day_amt_gap_' + 'of_' + x].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    every_day_amt_gap.columns = ['UID', 'every_day_amt_gap_sum_' + x, 'every_day_amt_gap_std_' + x, 'every_day_amt_gap_mean_' + x, 'every_day_amt_gap_min_' + x, 'every_day_amt_gap_max_' + x]\n",
    "    data = pd.merge(data, every_day_amt_gap, on='UID', how='left')\n",
    "\n",
    "    # 每天用该x(如ip)进行交易的钱数的rate\n",
    "    every_day_amt_rate = temp[[x, 'day', 'trans_amt']]\n",
    "    every_day_amt_rate['every_day_' + x + '_amt'] = every_day_amt_rate['trans_amt']\n",
    "    every_day_amt_rate = every_day_amt_rate.groupby([x, 'day'])['every_day_' + x + '_amt'].agg('sum').reset_index()\n",
    "    every_day_amt_rate = pd.merge(temp[['UID', x, 'day']], every_day_amt_rate, on=[x, 'day'], how='left')\n",
    "    every_day_amt_rate = every_day_amt_rate[['UID', 'every_day_' + x + '_amt']]\n",
    "    every_day_amt_rate['before_every_day_amt'] = every_day_amt_rate.groupby('UID')['every_day_' + x + '_amt'].shift(1)\n",
    "    every_day_amt_rate['every_day_amt_rate_' + 'of_' + x] = every_day_amt_rate['every_day_' + x + '_amt'] / every_day_amt_rate['before_every_day_amt']\n",
    "    every_day_amt_rate = every_day_amt_rate[['UID', 'every_day_amt_rate_' + 'of_' + x]]\n",
    "    every_day_amt_rate = every_day_amt_rate.groupby('UID')['every_day_amt_rate_' + 'of_' + x].agg('sum').reset_index()\n",
    "    data = pd.merge(data, every_day_amt_rate, on='UID', how='left')\n",
    "    \n",
    "    \n",
    "    \n",
    "    # 每天半夜用该x(如ip)进行交易的钱数\n",
    "    every_morning_amt = temp[temp.hour <= 5][[x, 'day', 'trans_amt']]\n",
    "    every_morning_amt['every_morning_' + x + '_amt'] = every_morning_amt['trans_amt']\n",
    "    every_morning_amt = every_morning_amt.groupby([x, 'day'])['every_morning_' + x + '_amt'].agg('sum').reset_index()\n",
    "    every_morning_amt = pd.merge(temp[['UID', x, 'day']], every_morning_amt, on=[x, 'day'], how='left')\n",
    "    every_morning_amt = every_morning_amt[['UID', 'every_morning_' + x + '_amt']]\n",
    "    every_morning_amt = every_morning_amt.groupby('UID')['every_morning_' + x + '_amt'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    every_morning_amt.columns = ['UID', 'every_morning_amt_std_' + x, 'every_morning_amt_mean_' + x, 'every_morning_amt_min_' + x, 'every_morning_amt_max_' + x]\n",
    "    data = pd.merge(data, every_morning_amt, on='UID', how='left')\n",
    "\n",
    "    # 每天半夜用该x(如ip)进行交易的钱数的gap\n",
    "    every_morning_amt_gap = temp[temp.hour <= 5][[x, 'day', 'trans_amt']]\n",
    "    every_morning_amt_gap['every_morning_' + x + '_amt'] = every_morning_amt_gap['trans_amt']\n",
    "    every_morning_amt_gap = every_morning_amt_gap.groupby([x, 'day'])['every_morning_' + x + '_amt'].agg('sum').reset_index()\n",
    "    every_morning_amt_gap = pd.merge(temp[['UID', x, 'day']], every_morning_amt_gap, on=[x, 'day'], how='left')\n",
    "    every_morning_amt_gap = every_morning_amt_gap[['UID', 'every_morning_' + x + '_amt']]\n",
    "    every_morning_amt_gap['before_every_morning_amt'] = every_morning_amt_gap.groupby('UID')['every_morning_' + x + '_amt'].shift(1)\n",
    "    every_morning_amt_gap['every_morning_amt_gap_' + 'of_' + x] = every_morning_amt_gap['every_morning_' + x + '_amt'] - every_morning_amt_gap['before_every_morning_amt']\n",
    "    every_morning_amt_gap = every_morning_amt_gap[['UID', 'every_morning_amt_gap_' + 'of_' + x]]\n",
    "    every_morning_amt_gap = every_morning_amt_gap.groupby('UID')['every_morning_amt_gap_' + 'of_' + x].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    every_morning_amt_gap.columns = ['UID', 'every_morning_amt_gap_sum_' + x, 'every_morning_amt_gap_std_' + x, 'every_morning_amt_gap_mean_' + x, 'every_morning_amt_gap_min_' + x, 'every_morning_amt_gap_max_' + x]\n",
    "    data = pd.merge(data, every_morning_amt_gap, on='UID', how='left')\n",
    "\n",
    "     # 每天半夜用该x(如ip)进行交易的钱数的rate\n",
    "    every_morning_amt_rate = temp[temp.hour <= 5][[x, 'day', 'trans_amt']]\n",
    "    every_morning_amt_rate['every_morning_' + x + '_amt'] = every_morning_amt_rate['trans_amt']\n",
    "    every_morning_amt_rate = every_morning_amt_rate.groupby([x, 'day'])['every_morning_' + x + '_amt'].agg('sum').reset_index()\n",
    "    every_morning_amt_rate = pd.merge(temp[['UID', x, 'day']], every_morning_amt_rate, on=[x, 'day'], how='left')\n",
    "    every_morning_amt_rate = every_morning_amt_rate[['UID', 'every_morning_' + x + '_amt']]\n",
    "    every_morning_amt_rate['before_every_morning_amt'] = every_morning_amt_rate.groupby('UID')['every_morning_' + x + '_amt'].shift(1)\n",
    "    every_morning_amt_rate['every_morning_amt_rate_' + 'of_' + x] = every_morning_amt_rate['every_morning_' + x + '_amt'] / every_morning_amt_rate['before_every_morning_amt']\n",
    "    every_morning_amt_rate = every_morning_amt_rate[['UID', 'every_morning_amt_rate_' + 'of_' + x]]\n",
    "    every_morning_amt_rate = every_morning_amt_rate.groupby('UID')['every_morning_amt_rate_' + 'of_' + x].agg('sum').reset_index()\n",
    "    data = pd.merge(data, every_morning_amt_rate, on='UID', how='left')\n",
    "    \n",
    "    \n",
    "    # 每天早上用该x(如ip)进行交易的钱数\n",
    "    every_work_time_amt = temp[(temp.hour <= 11)&(temp.hour >= 6)][[x, 'day', 'trans_amt']]\n",
    "    every_work_time_amt['every_work_time_' + x + '_amt'] = every_work_time_amt['trans_amt']\n",
    "    every_work_time_amt = every_work_time_amt.groupby([x, 'day'])['every_work_time_' + x + '_amt'].agg('sum').reset_index()\n",
    "    every_work_time_amt = pd.merge(temp[['UID', x, 'day']], every_work_time_amt, on=[x, 'day'], how='left')\n",
    "    every_work_time_amt = every_work_time_amt[['UID', 'every_work_time_' + x + '_amt']]\n",
    "    every_work_time_amt = every_work_time_amt.groupby('UID')['every_work_time_' + x + '_amt'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    every_work_time_amt.columns = ['UID', 'every_work_time_amt_std_' + x, 'every_work_time_amt_mean_' + x, 'every_work_time_amt_min_' + x, 'every_work_time_amt_max_' + x]\n",
    "    data = pd.merge(data, every_work_time_amt, on='UID', how='left')\n",
    "\n",
    "    # 每天早上用该x(如ip)进行交易的钱数的gap\n",
    "    every_work_time_amt_gap = temp[(temp.hour <= 11)&(temp.hour >= 6)][[x, 'day', 'trans_amt']]\n",
    "    every_work_time_amt_gap['every_work_time_' + x + '_amt'] = every_work_time_amt_gap['trans_amt']\n",
    "    every_work_time_amt_gap = every_work_time_amt_gap.groupby([x, 'day'])['every_work_time_' + x + '_amt'].agg('sum').reset_index()\n",
    "    every_work_time_amt_gap = pd.merge(temp[['UID', x, 'day']], every_work_time_amt_gap, on=[x, 'day'], how='left')\n",
    "    every_work_time_amt_gap = every_work_time_amt_gap[['UID', 'every_work_time_' + x + '_amt']]\n",
    "    every_work_time_amt_gap['before_every_work_time_amt'] = every_work_time_amt_gap.groupby('UID')['every_work_time_' + x + '_amt'].shift(1)\n",
    "    every_work_time_amt_gap['every_work_time_amt_gap_' + 'of_' + x] = every_work_time_amt_gap['every_work_time_' + x + '_amt'] - every_work_time_amt_gap['before_every_work_time_amt']\n",
    "    every_work_time_amt_gap = every_work_time_amt_gap[['UID', 'every_work_time_amt_gap_' + 'of_' + x]]\n",
    "    every_work_time_amt_gap = every_work_time_amt_gap.groupby('UID')['every_work_time_amt_gap_' + 'of_' + x].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    every_work_time_amt_gap.columns = ['UID', 'every_work_time_amt_gap_sum_' + x, 'every_work_time_amt_gap_std_' + x, 'every_work_time_amt_gap_mean_' + x, 'every_work_time_amt_gap_min_' + x, 'every_work_time_amt_gap_max_' + x]\n",
    "    data = pd.merge(data, every_work_time_amt_gap, on='UID', how='left')\n",
    "\n",
    "     # 每天早上用该x(如ip)进行交易的钱数的rate\n",
    "    every_work_time_amt_rate = temp[(temp.hour <= 11)&(temp.hour >= 6)][[x, 'day', 'trans_amt']]\n",
    "    every_work_time_amt_rate['every_work_time_' + x + '_amt'] = every_work_time_amt_rate['trans_amt']\n",
    "    every_work_time_amt_rate = every_work_time_amt_rate.groupby([x, 'day'])['every_work_time_' + x + '_amt'].agg('sum').reset_index()\n",
    "    every_work_time_amt_rate = pd.merge(temp[['UID', x, 'day']], every_work_time_amt_rate, on=[x, 'day'], how='left')\n",
    "    every_work_time_amt_rate = every_work_time_amt_rate[['UID', 'every_work_time_' + x + '_amt']]\n",
    "    every_work_time_amt_rate['before_every_work_time_amt'] = every_work_time_amt_rate.groupby('UID')['every_work_time_' + x + '_amt'].shift(1)\n",
    "    every_work_time_amt_rate['every_work_time_amt_rate_' + 'of_' + x] = every_work_time_amt_rate['every_work_time_' + x + '_amt'] / every_work_time_amt_rate['before_every_work_time_amt']\n",
    "    every_work_time_amt_rate = every_work_time_amt_rate[['UID', 'every_work_time_amt_rate_' + 'of_' + x]]\n",
    "    every_work_time_amt_rate = every_work_time_amt_rate.groupby('UID')['every_work_time_amt_rate_' + 'of_' + x].agg('sum').reset_index()\n",
    "    data = pd.merge(data, every_work_time_amt_rate, on='UID', how='left')\n",
    "    \n",
    "    # 每天下午用该x(如ip)进行交易的钱数\n",
    "    every_afternoon_amt = temp[(temp.hour <= 17)&(temp.hour >= 12)][[x, 'day', 'trans_amt']]\n",
    "    every_afternoon_amt['every_afternoon_' + x + '_amt'] = every_afternoon_amt['trans_amt']\n",
    "    every_afternoon_amt = every_afternoon_amt.groupby([x, 'day'])['every_afternoon_' + x + '_amt'].agg('sum').reset_index()\n",
    "    every_afternoon_amt = pd.merge(temp[['UID', x, 'day']], every_afternoon_amt, on=[x, 'day'], how='left')\n",
    "    every_afternoon_amt = every_afternoon_amt[['UID', 'every_afternoon_' + x + '_amt']]\n",
    "    every_afternoon_amt = every_afternoon_amt.groupby('UID')['every_afternoon_' + x + '_amt'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    every_afternoon_amt.columns = ['UID', 'every_afternoon_amt_std_' + x, 'every_afternoon_amt_mean_' + x, 'every_afternoon_amt_min_' + x, 'every_afternoon_amt_max_' + x]\n",
    "    data = pd.merge(data, every_afternoon_amt, on='UID', how='left')\n",
    "\n",
    "    # 每天下午用该x(如ip)进行交易的钱数的gap\n",
    "    every_afternoon_amt_gap = temp[(temp.hour <= 17)&(temp.hour >= 12)][[x, 'day', 'trans_amt']]\n",
    "    every_afternoon_amt_gap['every_afternoon_' + x + '_amt'] = every_afternoon_amt_gap['trans_amt']\n",
    "    every_afternoon_amt_gap = every_afternoon_amt_gap.groupby([x, 'day'])['every_afternoon_' + x + '_amt'].agg('sum').reset_index()\n",
    "    every_afternoon_amt_gap = pd.merge(temp[['UID', x, 'day']], every_afternoon_amt_gap, on=[x, 'day'], how='left')\n",
    "    every_afternoon_amt_gap = every_afternoon_amt_gap[['UID', 'every_afternoon_' + x + '_amt']]\n",
    "    every_afternoon_amt_gap['before_every_afternoon_amt'] = every_afternoon_amt_gap.groupby('UID')['every_afternoon_' + x + '_amt'].shift(1)\n",
    "    every_afternoon_amt_gap['every_afternoon_amt_gap_' + 'of_' + x] = every_afternoon_amt_gap['every_afternoon_' + x + '_amt'] - every_afternoon_amt_gap['before_every_afternoon_amt']\n",
    "    every_afternoon_amt_gap = every_afternoon_amt_gap[['UID', 'every_afternoon_amt_gap_' + 'of_' + x]]\n",
    "    every_afternoon_amt_gap = every_afternoon_amt_gap.groupby('UID')['every_afternoon_amt_gap_' + 'of_' + x].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    every_afternoon_amt_gap.columns = ['UID', 'every_afternoon_amt_gap_sum_' + x, 'every_afternoon_amt_gap_std_' + x, 'every_afternoon_amt_gap_mean_' + x, 'every_afternoon_amt_gap_min_' + x, 'every_afternoon_amt_gap_max_' + x]\n",
    "    data = pd.merge(data, every_afternoon_amt_gap, on='UID', how='left')\n",
    "\n",
    "     # 每天下午用该x(如ip)进行交易的钱数的rate\n",
    "    every_afternoon_amt_rate = temp[(temp.hour <= 17)&(temp.hour >= 12)][[x, 'day', 'trans_amt']]\n",
    "    every_afternoon_amt_rate['every_afternoon_' + x + '_amt'] = every_afternoon_amt_rate['trans_amt']\n",
    "    every_afternoon_amt_rate = every_afternoon_amt_rate.groupby([x, 'day'])['every_afternoon_' + x + '_amt'].agg('sum').reset_index()\n",
    "    every_afternoon_amt_rate = pd.merge(temp[['UID', x, 'day']], every_afternoon_amt_rate, on=[x, 'day'], how='left')\n",
    "    every_afternoon_amt_rate = every_afternoon_amt_rate[['UID', 'every_afternoon_' + x + '_amt']]\n",
    "    every_afternoon_amt_rate['before_every_afternoon_amt'] = every_afternoon_amt_rate.groupby('UID')['every_afternoon_' + x + '_amt'].shift(1)\n",
    "    every_afternoon_amt_rate['every_afternoon_amt_rate_' + 'of_' + x] = every_afternoon_amt_rate['every_afternoon_' + x + '_amt'] / every_afternoon_amt_rate['before_every_afternoon_amt']\n",
    "    every_afternoon_amt_rate = every_afternoon_amt_rate[['UID', 'every_afternoon_amt_rate_' + 'of_' + x]]\n",
    "    every_afternoon_amt_rate = every_afternoon_amt_rate.groupby('UID')['every_afternoon_amt_rate_' + 'of_' + x].agg('sum').reset_index()\n",
    "    data = pd.merge(data, every_afternoon_amt_rate, on='UID', how='left')\n",
    "    \n",
    "    # 每天晚上用该x(如ip)进行交易的钱数\n",
    "    every_night_amt = temp[(temp.hour <= 23)&(temp.hour >= 18)][[x, 'day', 'trans_amt']]\n",
    "    every_night_amt['every_night_' + x + '_amt'] = every_night_amt['trans_amt']\n",
    "    every_night_amt = every_night_amt.groupby([x, 'day'])['every_night_' + x + '_amt'].agg('sum').reset_index()\n",
    "    every_night_amt = pd.merge(temp[['UID', x, 'day']], every_night_amt, on=[x, 'day'], how='left')\n",
    "    every_night_amt = every_night_amt[['UID', 'every_night_' + x + '_amt']]\n",
    "    every_night_amt = every_night_amt.groupby('UID')['every_night_' + x + '_amt'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    every_night_amt.columns = ['UID', 'every_night_amt_std_' + x, 'every_night_amt_mean_' + x, 'every_night_amt_min_' + x, 'every_night_amt_max_' + x]\n",
    "    data = pd.merge(data, every_night_amt, on='UID', how='left')\n",
    "\n",
    "    # 每天晚上用该x(如ip)进行交易的钱数的gap\n",
    "    every_night_amt_gap = temp[(temp.hour <= 23)&(temp.hour >= 18)][[x, 'day', 'trans_amt']]\n",
    "    every_night_amt_gap['every_night_' + x + '_amt'] = every_night_amt_gap['trans_amt']\n",
    "    every_night_amt_gap = every_night_amt_gap.groupby([x, 'day'])['every_night_' + x + '_amt'].agg('sum').reset_index()\n",
    "    every_night_amt_gap = pd.merge(temp[['UID', x, 'day']], every_night_amt_gap, on=[x, 'day'], how='left')\n",
    "    every_night_amt_gap = every_night_amt_gap[['UID', 'every_night_' + x + '_amt']]\n",
    "    every_night_amt_gap['before_every_night_amt'] = every_night_amt_gap.groupby('UID')['every_night_' + x + '_amt'].shift(1)\n",
    "    every_night_amt_gap['every_night_amt_gap_' + 'of_' + x] = every_night_amt_gap['every_night_' + x + '_amt'] - every_night_amt_gap['before_every_night_amt']\n",
    "    every_night_amt_gap = every_night_amt_gap[['UID', 'every_night_amt_gap_' + 'of_' + x]]\n",
    "    every_night_amt_gap = every_night_amt_gap.groupby('UID')['every_night_amt_gap_' + 'of_' + x].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    every_night_amt_gap.columns = ['UID', 'every_night_amt_gap_sum_' + x, 'every_night_amt_gap_std_' + x, 'every_night_amt_gap_mean_' + x, 'every_night_amt_gap_min_' + x, 'every_night_amt_gap_max_' + x]\n",
    "    data = pd.merge(data, every_night_amt_gap, on='UID', how='left')\n",
    "\n",
    "     # 每天晚上用该x(如ip)进行交易的钱数的rate\n",
    "    every_night_amt_rate = temp[(temp.hour <= 23)&(temp.hour >= 18)][[x, 'day', 'trans_amt']]\n",
    "    every_night_amt_rate['every_night_' + x + '_amt'] = every_night_amt_rate['trans_amt']\n",
    "    every_night_amt_rate = every_night_amt_rate.groupby([x, 'day'])['every_night_' + x + '_amt'].agg('sum').reset_index()\n",
    "    every_night_amt_rate = pd.merge(temp[['UID', x, 'day']], every_night_amt_rate, on=[x, 'day'], how='left')\n",
    "    every_night_amt_rate = every_night_amt_rate[['UID', 'every_night_' + x + '_amt']]\n",
    "    every_night_amt_rate['before_every_night_amt'] = every_night_amt_rate.groupby('UID')['every_night_' + x + '_amt'].shift(1)\n",
    "    every_night_amt_rate['every_night_amt_rate_' + 'of_' + x] = every_night_amt_rate['every_night_' + x + '_amt'] / every_night_amt_rate['before_every_night_amt']\n",
    "    every_night_amt_rate = every_night_amt_rate[['UID', 'every_night_amt_rate_' + 'of_' + x]]\n",
    "    every_night_amt_rate = every_night_amt_rate.groupby('UID')['every_night_amt_rate_' + 'of_' + x].agg('sum').reset_index()\n",
    "    data = pd.merge(data, every_night_amt_rate, on='UID', how='left')\n",
    "    \n",
    "    # 每三天x与钱的gap\n",
    "    every_three_day_amt_gap = temp[[x, 'day', 'trans_amt']]\n",
    "    every_three_day_amt_gap['every_three_day_' + x + '_amt'] = every_three_day_amt_gap['trans_amt']\n",
    "    every_three_day_amt_gap = every_three_day_amt_gap.groupby([x, 'day'])['every_three_day_' + x + '_amt'].agg('sum').reset_index()\n",
    "    every_three_day_amt_gap = pd.merge(temp[['UID', x, 'day']], every_three_day_amt_gap, on=[x, 'day'], how='left')\n",
    "    every_three_day_amt_gap = every_three_day_amt_gap[['UID', 'every_three_day_' + x + '_amt']]\n",
    "    every_three_day_amt_gap['before_every_three_day_amt'] = every_three_day_amt_gap.groupby('UID')['every_three_day_' + x + '_amt'].shift(3)\n",
    "    every_three_day_amt_gap['every_three_day_amt_gap_' + 'of_' + x] = every_three_day_amt_gap['every_three_day_' + x + '_amt'] - every_three_day_amt_gap['before_every_three_day_amt']\n",
    "    every_three_day_amt_gap = every_three_day_amt_gap[['UID', 'every_three_day_amt_gap_' + 'of_' + x]]\n",
    "    every_three_day_amt_gap = every_three_day_amt_gap.groupby('UID')['every_three_day_amt_gap_' + 'of_' + x].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    every_three_day_amt_gap.columns = ['UID', 'every_three_day_amt_gap_sum_' + x, 'every_three_day_amt_gap_std_' + x, 'every_three_day_amt_gap_mean_' + x, 'every_three_day_amt_gap_min_' + x, 'every_three_day_amt_gap_max_' + x]\n",
    "    data = pd.merge(data, every_three_day_amt_gap, on='UID', how='left')\n",
    "    \n",
    "    every_three_day_amt_rate = temp[[x, 'day', 'trans_amt']]\n",
    "    every_three_day_amt_rate['every_three_day_' + x + '_amt'] = every_three_day_amt_rate['trans_amt']\n",
    "    every_three_day_amt_rate = every_three_day_amt_rate.groupby([x, 'day'])['every_three_day_' + x + '_amt'].agg('sum').reset_index()\n",
    "    every_three_day_amt_rate = pd.merge(temp[['UID', x, 'day']], every_three_day_amt_rate, on=[x, 'day'], how='left')\n",
    "    every_three_day_amt_rate = every_three_day_amt_rate[['UID', 'every_three_day_' + x + '_amt']]\n",
    "    every_three_day_amt_rate['before_every_three_day_amt'] = every_three_day_amt_rate.groupby('UID')['every_three_day_' + x + '_amt'].shift(3)\n",
    "    every_three_day_amt_rate['every_three_day_amt_rate_' + 'of_' + x] = every_three_day_amt_rate['every_three_day_' + x + '_amt'] / every_three_day_amt_rate['before_every_three_day_amt']\n",
    "    every_three_day_amt_rate = every_three_day_amt_rate[['UID', 'every_three_day_amt_rate_' + 'of_' + x]]\n",
    "    every_three_day_amt_rate = every_three_day_amt_rate.groupby('UID')['every_three_day_amt_rate_' + 'of_' + x].agg('sum').reset_index()\n",
    "    data = pd.merge(data, every_three_day_amt_rate, on='UID', how='left')\n",
    "    \n",
    "    # 每七天x与钱的gap\n",
    "    every_sixteen_day_amt_gap = temp[[x, 'day', 'trans_amt']]\n",
    "    every_sixteen_day_amt_gap['every_sixteen_day_' + x + '_amt'] = every_sixteen_day_amt_gap['trans_amt']\n",
    "    every_sixteen_day_amt_gap = every_sixteen_day_amt_gap.groupby([x, 'day'])['every_sixteen_day_' + x + '_amt'].agg('sum').reset_index()\n",
    "    every_sixteen_day_amt_gap = pd.merge(temp[['UID', x, 'day']], every_sixteen_day_amt_gap, on=[x, 'day'], how='left')\n",
    "    every_sixteen_day_amt_gap = every_sixteen_day_amt_gap[['UID', 'every_sixteen_day_' + x + '_amt']]\n",
    "    every_sixteen_day_amt_gap['before_every_sixteen_day_amt'] = every_sixteen_day_amt_gap.groupby('UID')['every_sixteen_day_' + x + '_amt'].shift(7)\n",
    "    every_sixteen_day_amt_gap['every_sixteen_day_amt_gap_' + 'of_' + x] = every_sixteen_day_amt_gap['every_sixteen_day_' + x + '_amt'] - every_sixteen_day_amt_gap['before_every_sixteen_day_amt']\n",
    "    every_sixteen_day_amt_gap = every_sixteen_day_amt_gap[['UID', 'every_sixteen_day_amt_gap_' + 'of_' + x]]\n",
    "    every_sixteen_day_amt_gap = every_sixteen_day_amt_gap.groupby('UID')['every_sixteen_day_amt_gap_' + 'of_' + x].agg({'sum' ,'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    every_sixteen_day_amt_gap.columns = ['UID', 'every_sixteen_day_amt_gap_sum_' + x, 'every_sixteen_day_amt_gap_std_' + x, 'every_sixteen_day_amt_gap_mean_' + x, 'every_sixteen_day_amt_gap_min_' + x, 'every_sixteen_day_amt_gap_max_' + x]\n",
    "    data = pd.merge(data, every_sixteen_day_amt_gap, on='UID', how='left')\n",
    "\n",
    "    every_sixteen_day_amt_rate = temp[[x, 'day', 'trans_amt']]\n",
    "    every_sixteen_day_amt_rate['every_sixteen_day_' + x + '_amt'] = every_sixteen_day_amt_rate['trans_amt']\n",
    "    every_sixteen_day_amt_rate = every_sixteen_day_amt_rate.groupby([x, 'day'])['every_sixteen_day_' + x + '_amt'].agg('sum').reset_index()\n",
    "    every_sixteen_day_amt_rate = pd.merge(temp[['UID', x, 'day']], every_sixteen_day_amt_rate, on=[x, 'day'], how='left')\n",
    "    every_sixteen_day_amt_rate = every_sixteen_day_amt_rate[['UID', 'every_sixteen_day_' + x + '_amt']]\n",
    "    every_sixteen_day_amt_rate['before_every_sixteen_day_amt'] = every_sixteen_day_amt_rate.groupby('UID')['every_sixteen_day_' + x + '_amt'].shift(7)\n",
    "    every_sixteen_day_amt_rate['every_sixteen_day_amt_rate_' + 'of_' + x] = every_sixteen_day_amt_rate['every_sixteen_day_' + x + '_amt'] / every_sixteen_day_amt_rate['before_every_sixteen_day_amt']\n",
    "    every_sixteen_day_amt_rate = every_sixteen_day_amt_rate[['UID', 'every_sixteen_day_amt_rate_' + 'of_' + x]]\n",
    "    every_sixteen_day_amt_rate = every_sixteen_day_amt_rate.groupby('UID')['every_sixteen_day_amt_rate_' + 'of_' + x].agg('sum').reset_index()\n",
    "    data = pd.merge(data, every_sixteen_day_amt_rate, on='UID', how='left')\n",
    "    \n",
    "    print(x + ' is over!')\n",
    "    return data\n",
    "\n",
    "for x in ['merchant', 'mac1', 'ip1_sub', 'market_code']:\n",
    "    data = amt_label_frequence(data, transaction_data, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data['UID_'] = data['UID']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 每一个用户的交易总金额\n",
    "amt = transaction_data[['UID', 'trans_amt']]\n",
    "amt['trans_amt_sum'] = amt['trans_amt']\n",
    "amt = amt.groupby('UID')['trans_amt_sum'].agg('sum').reset_index()\n",
    "data = pd.merge(data, amt, on='UID', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_label_uid_own(data, operation_or_transaction_data, label, t):\n",
    "    label_count = operation_or_transaction_data[['UID', label]]\n",
    "    operation_or_transaction_data = label_count.copy()\n",
    "    operation_or_transaction_data.drop_duplicates(inplace=True)\n",
    "    \n",
    "    label_count[label + '_uid_count_' + t] = label_count['UID']\n",
    "    label_count = label_count.groupby([label])[label + '_uid_count_' + t].agg('nunique').reset_index()# 好气啊！=_=\n",
    "    \n",
    "    operation_or_transaction_data = pd.merge(operation_or_transaction_data, label_count, on=label, how='left')\n",
    "    operation_or_transaction_data[label + '_count_uid_' + t] = 1\n",
    "    operation_or_transaction_data = operation_or_transaction_data.groupby('UID')[label + '_uid_count_' + t].agg('sum').reset_index()\n",
    "    \n",
    "    data = pd.merge(data, operation_or_transaction_data, on='UID', how='left')\n",
    "    \n",
    "    return data\n",
    "\n",
    "# 每一个用户uid的每一种label种类数\n",
    "for x in ['day', 'success', 'device_code1', 'device_code2', 'mac1', 'ip1', 'ip2', 'device_code3', 'mac2', 'wifi', 'geo_code', 'ip1_sub', 'ip2_sub']:\n",
    "    data = get_label_uid_own(data, operation_data, x, 'operation')\n",
    "for x in ['day', 'merchant', 'code1', 'code2', 'acc_id1', 'device_code1',  'device_code2', 'device_code3', 'mac1', 'ip1', 'acc_id2', 'acc_id3', 'geo_code', 'ip1_sub', 'trans_amt']:\n",
    "    data = get_label_uid_own(data, transaction_data, x, 'transaction')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['device2',\n",
       " 'ip1_sub',\n",
       " 'device_code2',\n",
       " 'ip1',\n",
       " 'geo_code',\n",
       " 'device_code3',\n",
       " 'day',\n",
       " 'device_code1',\n",
       " 'mac1',\n",
       " 'device1']"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_same_label(data, operation_data, transaction_data):\n",
    "    not_need = ['time', 'UID', 'hour']\n",
    "    operation = [x for x in operation_data if x not in not_need]\n",
    "    transaction = [x for x in transaction_data if x not in not_need]\n",
    "    \n",
    "    same_label = list(set(operation).intersection(set(transaction)))\n",
    "    return same_label\n",
    "same_label = get_same_label(data, operation_data, transaction_data)\n",
    "same_label.remove('minute')\n",
    "same_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "device2 over!\n",
      "ip1_sub over!\n",
      "device_code2 over!\n",
      "ip1 over!\n",
      "geo_code over!\n",
      "device_code3 over!\n",
      "day over!\n",
      "device_code1 over!\n",
      "mac1 over!\n",
      "device1 over!\n"
     ]
    }
   ],
   "source": [
    "def get_uid_operation_transaction_different(data, operation_data, transaction_data, same_label):\n",
    "    label = same_label.copy()\n",
    "    label.append('UID')\n",
    "    \n",
    "    operation = operation_data[label]\n",
    "    transaction = transaction_data[label]\n",
    "\n",
    "    temp = pd.concat([operation, transaction], axis=0, ignore_index=True)# 先堆在一起节约运行时间\n",
    "    \n",
    "    for x in same_label:\n",
    "        t = temp[['UID', x]]\n",
    "        t['same'] = t[x]\n",
    "        t = t.groupby('UID')['same'].agg('nunique').reset_index()\n",
    "        \n",
    "        t1 = operation[['UID', x]]\n",
    "        t1['operation'] = t1[x]\n",
    "        t1 = t1.groupby('UID')['operation'].agg('nunique').reset_index()\n",
    "\n",
    "        t2 = transaction[['UID', x]]\n",
    "        t2['transaction'] = t2[x]\n",
    "        t2 = t2.groupby('UID')['transaction'].agg('nunique').reset_index()\n",
    "\n",
    "        t = pd.merge(t, t1, on='UID', how='left')\n",
    "        t = pd.merge(t, t2, on='UID', how='left')\n",
    "        t = t.fillna(0)\n",
    "        t['uid_operation_transaction_different_of_' + x] = t['operation'] + t['transaction'] - t['same']\n",
    "        t = t.drop(['operation', 'transaction', 'same'], axis=1)\n",
    "        data = pd.merge(data, t, on='UID', how='left')\n",
    "        print(x + ' over!')\n",
    "        \n",
    "    return data\n",
    "\n",
    "# 统计每一个用户uid操作和交易的每一种label不相同的数量\n",
    "data = get_uid_operation_transaction_different(data, operation_data, transaction_data, same_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_is_null_ratio(data, t_data, x, t):\n",
    "    \"\"\"\n",
    "    对每一个UID的每一个设备类型，计算nan值的count\n",
    "    \"\"\"\n",
    "    temp = t_data[t_data[x].isnull()][['UID']]\n",
    "    temp[x + '_null_' + t] = 1\n",
    "    temp = temp.groupby('UID').agg('count').reset_index()\n",
    "    temp = temp[['UID', x + '_null_' + t]]\n",
    "    data = pd.merge(data, temp, on='UID', how='left')\n",
    "    return data\n",
    "\n",
    "o = ['success', 'version', 'device1', 'device2', 'device_code1', 'device_code2', 'device_code3', 'mac1', 'mac2', 'ip1', 'ip2', 'geo_code', 'ip1_sub', 'ip2_sub']\n",
    "t = ['code1', 'code2', 'acc_id1', 'device_code1', 'device_code2', 'device_code3', 'device1', 'device2', 'mac1', 'ip1', 'amt_src2', 'acc_id2', 'acc_id3', 'geo_code', 'trans_type2', 'market_code', 'market_type', 'ip1_sub']\n",
    "for x in o:\n",
    "    data = get_is_null_ratio(data, operation_data, x, 'operation')\n",
    "for y in t:\n",
    "    data = get_is_null_ratio(data, transaction_data, y, 'transaction')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "支付方式\n",
    "trans_type1是真正的方式，trans_type2是线上线下这种\n",
    "\"\"\"\n",
    "# 每一天的每一小时用户支付方式的支付次数\n",
    "trans_type1_count = transaction_data[['UID', 'day', 'hour', 'trans_type1']]\n",
    "trans_type1_count['every_hour_trans_type1_count'] = 1\n",
    "trans_type1_count = trans_type1_count.groupby(['UID', 'day', 'hour', 'trans_type1']).agg('count').reset_index()\n",
    "trans_type1_count = trans_type1_count[['UID', 'every_hour_trans_type1_count']]\n",
    "trans_type1_count = trans_type1_count.groupby('UID')['every_hour_trans_type1_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "trans_type1_count.columns = ['UID', 'trans_type1_count_mean', 'trans_type1_count_min', 'trans_type1_count_std', 'trans_type1_count_max']\n",
    "data = pd.merge(data, trans_type1_count, on='UID', how='left')\n",
    "\n",
    "# 每一天每一小时用户在该ip用该支付方式支付次数\n",
    "trans_type1_ip1_count = transaction_data[['UID', 'day', 'hour', 'ip1_sub', 'trans_type1']]\n",
    "trans_type1_ip1_count['every_hour_ip1_sub_trans_type1_count'] = 1\n",
    "trans_type1_ip1_count = trans_type1_ip1_count.groupby(['UID', 'day', 'hour', 'ip1_sub', 'trans_type1']).agg('count').reset_index()\n",
    "trans_type1_ip1_count = trans_type1_ip1_count[['UID', 'every_hour_ip1_sub_trans_type1_count']]\n",
    "trans_type1_ip1_count = trans_type1_ip1_count.groupby('UID')['every_hour_ip1_sub_trans_type1_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "trans_type1_ip1_count.columns = ['UID', 'trans_type1_ip1_count_mean', 'trans_type1_ip1_count_min', 'trans_type1_ip1_count_std', 'trans_type1_ip1_count_max']\n",
    "data = pd.merge(data, trans_type1_ip1_count, on='UID', how='left')\n",
    "\n",
    "# 每一天每一小时用户在该ip用该支付方式支付次数\n",
    "trans_type1_mac1_count = transaction_data[['UID', 'day', 'hour', 'mac1', 'trans_type1']]\n",
    "trans_type1_mac1_count['every_hour_mac1_trans_type1_count'] = 1\n",
    "trans_type1_mac1_count = trans_type1_mac1_count.groupby(['UID', 'day', 'hour', 'mac1', 'trans_type1']).agg('count').reset_index()\n",
    "trans_type1_mac1_count = trans_type1_mac1_count[['UID', 'every_hour_mac1_trans_type1_count']]\n",
    "trans_type1_mac1_count = trans_type1_mac1_count.groupby('UID')['every_hour_mac1_trans_type1_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "trans_type1_mac1_count.columns = ['UID', 'trans_type1_mac1_count_mean', 'trans_type1_mac1_count_min', 'trans_type1_mac1_count_std', 'trans_type1_mac1_count_max']\n",
    "data = pd.merge(data, trans_type1_mac1_count, on='UID', how='left')\n",
    "\n",
    "# 每一天每一小时用户用device1用该支付方式支付次数\n",
    "trans_type_device_code1_count = transaction_data[['UID', 'day', 'hour', 'device_code1', 'trans_type1']]\n",
    "trans_type_device_code1_count['every_hour_device_code1_trans_type1_count'] = 1\n",
    "trans_type_device_code1_count = trans_type_device_code1_count.groupby(['UID', 'day', 'hour', 'device_code1', 'trans_type1']).agg('count').reset_index()\n",
    "trans_type_device_code1_count = trans_type_device_code1_count[['UID', 'every_hour_device_code1_trans_type1_count']]\n",
    "trans_type_device_code1_count = trans_type_device_code1_count.groupby('UID')['every_hour_device_code1_trans_type1_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "trans_type_device_code1_count.columns = ['UID', 'trans_type_device_code1_count_mean', 'trans_type_device_code1_count_min', 'trans_type_device_code1_count_std', 'trans_type_device_code1_count_max']\n",
    "data = pd.merge(data, trans_type_device_code1_count, on='UID', how='left')\n",
    "\n",
    "# 每一天每一小时用户用device2用该支付方式支付次数\n",
    "trans_type_device_code2_count = transaction_data[['UID', 'day', 'hour', 'device_code2', 'trans_type1']]\n",
    "trans_type_device_code2_count['every_hour_device_code2_trans_type1_count'] = 1\n",
    "trans_type_device_code2_count = trans_type_device_code2_count.groupby(['UID', 'day', 'hour', 'device_code2', 'trans_type1']).agg('count').reset_index()\n",
    "trans_type_device_code2_count = trans_type_device_code2_count[['UID', 'every_hour_device_code2_trans_type1_count']]\n",
    "trans_type_device_code2_count = trans_type_device_code2_count.groupby('UID')['every_hour_device_code2_trans_type1_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "trans_type_device_code2_count.columns = ['UID', 'trans_type_device_code2_count_mean', 'trans_type_device_code2_count_min', 'trans_type_device_code2_count_std', 'trans_type_device_code2_count_max']\n",
    "data = pd.merge(data, trans_type_device_code2_count, on='UID', how='left')\n",
    "\n",
    "# 每一天每一小时用户用device3用该支付方式支付次数\n",
    "trans_type_device_code3_count = transaction_data[['UID', 'day', 'hour', 'device_code3', 'trans_type1']]\n",
    "trans_type_device_code3_count['every_hour_device_code3_trans_type1_count'] = 1\n",
    "trans_type_device_code3_count = trans_type_device_code3_count.groupby(['UID', 'day', 'hour', 'device_code3', 'trans_type1']).agg('count').reset_index()\n",
    "trans_type_device_code3_count = trans_type_device_code3_count[['UID', 'every_hour_device_code3_trans_type1_count']]\n",
    "trans_type_device_code3_count = trans_type_device_code3_count.groupby('UID')['every_hour_device_code3_trans_type1_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "trans_type_device_code3_count.columns = ['UID', 'trans_type_device_code3_count_mean', 'trans_type_device_code3_count_min', 'trans_type_device_code3_count_std', 'trans_type_device_code3_count_max']\n",
    "data = pd.merge(data, trans_type_device_code3_count, on='UID', how='left')\n",
    "\n",
    "# 每一天每一个月用户该支付方式的gap\n",
    "trans_type_hour_gap = transaction_data[['UID', 'day', 'hour', 'trans_type1']]\n",
    "trans_type_hour_gap['trans_type_hour_gap'] = 1\n",
    "trans_type_hour_gap = trans_type_hour_gap.groupby(['UID', 'day', 'hour', 'trans_type1']).agg('count').reset_index()\n",
    "trans_type_hour_gap['before_hour_gap'] = trans_type_hour_gap.groupby('UID')['trans_type_hour_gap'].shift(1)\n",
    "trans_type_hour_gap['trans_type_every_hour_gap'] = trans_type_hour_gap['trans_type_hour_gap'] - trans_type_hour_gap['before_hour_gap']\n",
    "trans_type_hour_gap = trans_type_hour_gap[['UID', 'trans_type_every_hour_gap']]\n",
    "trans_type_hour_gap = trans_type_hour_gap.groupby('UID')['trans_type_every_hour_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "trans_type_hour_gap.columns = ['UID', 'trans_type_hour_gap_sum', 'trans_type_hour_gap_mean', 'trans_type_hour_gap_min', 'trans_type_hour_gap_std', 'trans_type_hour_gap_max']\n",
    "data = pd.merge(data, trans_type_hour_gap, on='UID', how='left')\n",
    "\n",
    "# 每一天每一个月用户该支付方式的rate\n",
    "trans_type_hour_rate = transaction_data[['UID', 'day', 'hour', 'trans_type1']]\n",
    "trans_type_hour_rate['trans_type_hour_rate'] = 1\n",
    "trans_type_hour_rate = trans_type_hour_rate.groupby(['UID', 'day', 'hour', 'trans_type1']).agg('count').reset_index()\n",
    "trans_type_hour_rate['before_hour_rate'] = trans_type_hour_rate.groupby('UID')['trans_type_hour_rate'].shift(1)\n",
    "trans_type_hour_rate['trans_type_every_hour_rate'] = trans_type_hour_rate['trans_type_hour_rate'] / trans_type_hour_rate['before_hour_rate']\n",
    "trans_type_hour_rate = trans_type_hour_rate[['UID', 'trans_type_every_hour_rate']]\n",
    "trans_type_hour_rate = trans_type_hour_rate.groupby('UID')['trans_type_every_hour_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, trans_type_hour_rate, on='UID', how='left')\n",
    "\n",
    "\n",
    "# 每一天的用户支付方式的支付次数\n",
    "trans_type1_count = transaction_data[['UID', 'day', 'trans_type1']]\n",
    "trans_type1_count['every_day_trans_type1_count'] = 1\n",
    "trans_type1_count = trans_type1_count.groupby(['UID', 'day', 'trans_type1']).agg('count').reset_index()\n",
    "trans_type1_count = trans_type1_count[['UID', 'every_day_trans_type1_count']]\n",
    "trans_type1_count = trans_type1_count.groupby('UID')['every_day_trans_type1_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "trans_type1_count.columns = ['UID', 'trans_type1_count_mean_day', 'trans_type1_count_min_day', 'trans_type1_count_std_day', 'trans_type1_count_max_day']\n",
    "data = pd.merge(data, trans_type1_count, on='UID', how='left')\n",
    "\n",
    "# 每一天用户在该ip用该支付方式支付次数\n",
    "trans_type1_ip1_count = transaction_data[['UID', 'day', 'ip1_sub', 'trans_type1']]\n",
    "trans_type1_ip1_count['every_day_ip1_sub_trans_type1_count'] = 1\n",
    "trans_type1_ip1_count = trans_type1_ip1_count.groupby(['UID', 'day', 'ip1_sub', 'trans_type1']).agg('count').reset_index()\n",
    "trans_type1_ip1_count = trans_type1_ip1_count[['UID', 'every_day_ip1_sub_trans_type1_count']]\n",
    "trans_type1_ip1_count = trans_type1_ip1_count.groupby('UID')['every_day_ip1_sub_trans_type1_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "trans_type1_ip1_count.columns = ['UID', 'trans_type1_ip1_count_mean_day', 'trans_type1_ip1_count_min_day', 'trans_type1_ip1_count_std_day', 'trans_type1_ip1_count_max_day']\n",
    "data = pd.merge(data, trans_type1_ip1_count, on='UID', how='left')\n",
    "\n",
    "# 每一天用户在该mac用该支付方式支付次数\n",
    "trans_type1_mac1_count = transaction_data[['UID', 'day', 'mac1', 'trans_type1']]\n",
    "trans_type1_mac1_count['every_day_mac1_trans_type1_count'] = 1\n",
    "trans_type1_mac1_count = trans_type1_mac1_count.groupby(['UID', 'day', 'mac1', 'trans_type1']).agg('count').reset_index()\n",
    "trans_type1_mac1_count = trans_type1_mac1_count[['UID', 'every_day_mac1_trans_type1_count']]\n",
    "trans_type1_mac1_count = trans_type1_mac1_count.groupby('UID')['every_day_mac1_trans_type1_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "trans_type1_mac1_count.columns = ['UID', 'trans_type1_mac1_count_mean_day', 'trans_type1_mac1_count_min_day', 'trans_type1_mac1_count_std_day', 'trans_type1_mac1_count_max_day']\n",
    "data = pd.merge(data, trans_type1_mac1_count, on='UID', how='left')\n",
    "\n",
    "# 每一天用户用device1用该支付方式支付次数\n",
    "trans_type_device_code1_count = transaction_data[['UID', 'day', 'device_code1', 'trans_type1']]\n",
    "trans_type_device_code1_count['every_day_device_code1_trans_type1_count'] = 1\n",
    "trans_type_device_code1_count = trans_type_device_code1_count.groupby(['UID', 'day', 'device_code1', 'trans_type1']).agg('count').reset_index()\n",
    "trans_type_device_code1_count = trans_type_device_code1_count[['UID', 'every_day_device_code1_trans_type1_count']]\n",
    "trans_type_device_code1_count = trans_type_device_code1_count.groupby('UID')['every_day_device_code1_trans_type1_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "trans_type_device_code1_count.columns = ['UID', 'trans_type_device_code1_count_mean_day', 'trans_type_device_code1_count_min_day', 'trans_type_device_code1_count_std_day', 'trans_type_device_code1_count_max_day']\n",
    "data = pd.merge(data, trans_type_device_code1_count, on='UID', how='left')\n",
    "\n",
    "# 每一天用户用device2用该支付方式支付次数\n",
    "trans_type_device_code2_count = transaction_data[['UID', 'day', 'device_code2', 'trans_type1']]\n",
    "trans_type_device_code2_count['every_day_device_code2_trans_type1_count'] = 1\n",
    "trans_type_device_code2_count = trans_type_device_code2_count.groupby(['UID', 'day', 'device_code2', 'trans_type1']).agg('count').reset_index()\n",
    "trans_type_device_code2_count = trans_type_device_code2_count[['UID', 'every_day_device_code2_trans_type1_count']]\n",
    "trans_type_device_code2_count = trans_type_device_code2_count.groupby('UID')['every_day_device_code2_trans_type1_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "trans_type_device_code2_count.columns = ['UID', 'trans_type_device_code2_count_mean_day', 'trans_type_device_code2_count_min_day', 'trans_type_device_code2_count_std_day', 'trans_type_device_code2_count_max_day']\n",
    "data = pd.merge(data, trans_type_device_code2_count, on='UID', how='left')\n",
    "\n",
    "# 每一天用户用device3用该支付方式支付次数\n",
    "trans_type_device_code3_count = transaction_data[['UID', 'day', 'device_code3', 'trans_type1']]\n",
    "trans_type_device_code3_count['every_day_device_code3_trans_type1_count'] = 1\n",
    "trans_type_device_code3_count = trans_type_device_code3_count.groupby(['UID', 'day', 'device_code3', 'trans_type1']).agg('count').reset_index()\n",
    "trans_type_device_code3_count = trans_type_device_code3_count[['UID', 'every_day_device_code3_trans_type1_count']]\n",
    "trans_type_device_code3_count = trans_type_device_code3_count.groupby('UID')['every_day_device_code3_trans_type1_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "trans_type_device_code3_count.columns = ['UID', 'trans_type_device_code3_count_mean_day', 'trans_type_device_code3_count_min_day', 'trans_type_device_code3_count_std_day', 'trans_type_device_code3_count_max_day']\n",
    "data = pd.merge(data, trans_type_device_code3_count, on='UID', how='left')\n",
    "\n",
    "# 每一天每一个月用户该支付方式的gap\n",
    "trans_type_day_gap = transaction_data[['UID', 'day', 'trans_type1']]\n",
    "trans_type_day_gap['trans_type_day_gap'] = 1\n",
    "trans_type_day_gap = trans_type_day_gap.groupby(['UID', 'day', 'trans_type1']).agg('count').reset_index()\n",
    "trans_type_day_gap['before_day_gap'] = trans_type_day_gap.groupby('UID')['trans_type_day_gap'].shift(1)\n",
    "trans_type_day_gap['trans_type_every_day_gap'] = trans_type_day_gap['trans_type_day_gap'] - trans_type_day_gap['before_day_gap']\n",
    "trans_type_day_gap = trans_type_day_gap[['UID', 'trans_type_every_day_gap']]\n",
    "trans_type_day_gap = trans_type_day_gap.groupby('UID')['trans_type_every_day_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "trans_type_day_gap.columns = ['UID', 'trans_type_day_gap_sum_day', 'trans_type_day_gap_mean_day', 'trans_type_day_gap_min_day', 'trans_type_day_gap_std_day', 'trans_type_day_gap_max_day']\n",
    "data = pd.merge(data, trans_type_day_gap, on='UID', how='left')\n",
    "\n",
    "# 每一天每一个月用户该支付方式的rate\n",
    "trans_type_day_rate = transaction_data[['UID', 'day', 'trans_type1']]\n",
    "trans_type_day_rate['trans_type_day_rate'] = 1\n",
    "trans_type_day_rate = trans_type_day_rate.groupby(['UID', 'day', 'trans_type1']).agg('count').reset_index()\n",
    "trans_type_day_rate['before_day_rate'] = trans_type_day_rate.groupby('UID')['trans_type_day_rate'].shift(1)\n",
    "trans_type_day_rate['trans_type_every_day_rate'] = trans_type_day_rate['trans_type_day_rate'] / trans_type_day_rate['before_day_rate']\n",
    "trans_type_day_rate = trans_type_day_rate[['UID', 'trans_type_every_day_rate']]\n",
    "trans_type_day_rate = trans_type_day_rate.groupby('UID')['trans_type_every_day_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, trans_type_day_rate, on='UID', how='left')\n",
    "\n",
    "\n",
    "# 每天的半夜用户该支付方式的count\n",
    "every_morning_trans_type_count = transaction_data[transaction_data.hour <= 5][['UID', 'day', 'hour', 'trans_type1']]\n",
    "every_morning_trans_type_count['every_morning_trans_type_count'] = 1\n",
    "every_morning_trans_type_count = every_morning_trans_type_count.groupby(['UID', 'day', 'hour', 'trans_type1']).agg('count').reset_index()\n",
    "every_morning_trans_type_count = every_morning_trans_type_count[['UID', 'every_morning_trans_type_count']]\n",
    "every_morning_trans_type_count = every_morning_trans_type_count.groupby('UID')['every_morning_trans_type_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_morning_trans_type_count.columns = ['UID', 'every_morning_trans_type_count_mean', 'every_morning_trans_type_count_min', 'every_morning_trans_type_count_std', 'every_morning_trans_type_count_max']\n",
    "data = pd.merge(data, every_morning_trans_type_count, on='UID', how='left')\n",
    "\n",
    "# 每天的半夜用户该支付方式的gap\n",
    "every_morning_trans_type_gap = transaction_data[transaction_data.hour <= 5][['UID', 'day', 'hour', 'trans_type1']]\n",
    "every_morning_trans_type_gap['every_morning_trans_type_gap'] = 1\n",
    "every_morning_trans_type_gap = every_morning_trans_type_gap.groupby(['UID', 'day', 'hour', 'trans_type1']).agg('count').reset_index()\n",
    "every_morning_trans_type_gap = every_morning_trans_type_gap[['UID', 'every_morning_trans_type_gap']]\n",
    "every_morning_trans_type_gap['before_every_morning'] = every_morning_trans_type_gap.groupby('UID')['every_morning_trans_type_gap'].shift(1)\n",
    "every_morning_trans_type_gap['morning_trans_type_gap'] = every_morning_trans_type_gap['every_morning_trans_type_gap'] - every_morning_trans_type_gap['before_every_morning']\n",
    "every_morning_trans_type_gap = every_morning_trans_type_gap[['UID', 'morning_trans_type_gap']]\n",
    "every_morning_trans_type_gap = every_morning_trans_type_gap.groupby('UID')['morning_trans_type_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_morning_trans_type_gap.columns = ['UID', 'every_morning_trans_type_gap_sum', 'every_morning_trans_type_gap_mean', 'every_morning_trans_type_gap_min', 'every_morning_trans_type_gap_std', 'every_morning_trans_type_gap_max']\n",
    "data = pd.merge(data, every_morning_trans_type_gap, on='UID', how='left')\n",
    "\n",
    "# 每天的半夜用户该支付方式的rate\n",
    "every_morning_trans_type_rate = transaction_data[transaction_data.hour <= 5][['UID', 'day', 'hour', 'trans_type1']]\n",
    "every_morning_trans_type_rate['every_morning_trans_type_rate'] = 1\n",
    "every_morning_trans_type_rate = every_morning_trans_type_rate.groupby(['UID', 'day', 'hour', 'trans_type1']).agg('count').reset_index()\n",
    "every_morning_trans_type_rate = every_morning_trans_type_rate[['UID', 'every_morning_trans_type_rate']]\n",
    "every_morning_trans_type_rate['before_every_morning'] = every_morning_trans_type_rate.groupby('UID')['every_morning_trans_type_rate'].shift(1)\n",
    "every_morning_trans_type_rate['morning_trans_type_rate'] = every_morning_trans_type_rate['every_morning_trans_type_rate'] / every_morning_trans_type_rate['before_every_morning']\n",
    "every_morning_trans_type_rate = every_morning_trans_type_rate[['UID', 'morning_trans_type_rate']]\n",
    "every_morning_trans_type_rate = every_morning_trans_type_rate.groupby('UID')['morning_trans_type_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, every_morning_trans_type_rate, on='UID', how='left')\n",
    "\n",
    "# 每天的早上用户该支付方式的count\n",
    "every_work_time_trans_type_count = transaction_data[(transaction_data.hour >= 6)&(transaction_data.hour <= 11)][['UID', 'day', 'hour', 'trans_type1']]\n",
    "every_work_time_trans_type_count['every_work_time_trans_type_count'] = 1\n",
    "every_work_time_trans_type_count = every_work_time_trans_type_count.groupby(['UID', 'day', 'hour', 'trans_type1']).agg('count').reset_index()\n",
    "every_work_time_trans_type_count = every_work_time_trans_type_count[['UID', 'every_work_time_trans_type_count']]\n",
    "every_work_time_trans_type_count = every_work_time_trans_type_count.groupby('UID')['every_work_time_trans_type_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_work_time_trans_type_count.columns = ['UID', 'every_work_time_trans_type_count_mean', 'every_work_time_trans_type_count_min', 'every_work_time_trans_type_count_std', 'every_work_time_trans_type_count_max']\n",
    "data = pd.merge(data, every_work_time_trans_type_count, on='UID', how='left')\n",
    "\n",
    "# 每天的早上用户该支付方式的gap\n",
    "every_work_time_trans_type_gap = transaction_data[(transaction_data.hour >= 6)&(transaction_data.hour <= 11)][['UID', 'day', 'hour', 'trans_type1']]\n",
    "every_work_time_trans_type_gap['every_work_time_trans_type_gap'] = 1\n",
    "every_work_time_trans_type_gap = every_work_time_trans_type_gap.groupby(['UID', 'day', 'hour', 'trans_type1']).agg('count').reset_index()\n",
    "every_work_time_trans_type_gap = every_work_time_trans_type_gap[['UID', 'every_work_time_trans_type_gap']]\n",
    "every_work_time_trans_type_gap['before_every_work_time'] = every_work_time_trans_type_gap.groupby('UID')['every_work_time_trans_type_gap'].shift(1)\n",
    "every_work_time_trans_type_gap['work_time_trans_type_gap'] = every_work_time_trans_type_gap['every_work_time_trans_type_gap'] - every_work_time_trans_type_gap['before_every_work_time']\n",
    "every_work_time_trans_type_gap = every_work_time_trans_type_gap[['UID', 'work_time_trans_type_gap']]\n",
    "every_work_time_trans_type_gap = every_work_time_trans_type_gap.groupby('UID')['work_time_trans_type_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_work_time_trans_type_gap.columns = ['UID', 'every_work_time_trans_type_gap_sum', 'every_work_time_trans_type_gap_mean', 'every_work_time_trans_type_gap_min', 'every_work_time_trans_type_gap_std', 'every_work_time_trans_type_gap_max']\n",
    "data = pd.merge(data, every_work_time_trans_type_gap, on='UID', how='left')\n",
    "\n",
    "# 每天的早上用户该支付方式的rate\n",
    "every_work_time_trans_type_rate = transaction_data[(transaction_data.hour >= 6)&(transaction_data.hour <= 11)][['UID', 'day', 'hour', 'trans_type1']]\n",
    "every_work_time_trans_type_rate['every_work_time_trans_type_rate'] = 1\n",
    "every_work_time_trans_type_rate = every_work_time_trans_type_rate.groupby(['UID', 'day', 'hour', 'trans_type1']).agg('count').reset_index()\n",
    "every_work_time_trans_type_rate = every_work_time_trans_type_rate[['UID', 'every_work_time_trans_type_rate']]\n",
    "every_work_time_trans_type_rate['before_every_work_time'] = every_work_time_trans_type_rate.groupby('UID')['every_work_time_trans_type_rate'].shift(1)\n",
    "every_work_time_trans_type_rate['work_time_trans_type_rate'] = every_work_time_trans_type_rate['every_work_time_trans_type_rate'] / every_work_time_trans_type_rate['before_every_work_time']\n",
    "every_work_time_trans_type_rate = every_work_time_trans_type_rate[['UID', 'work_time_trans_type_rate']]\n",
    "every_work_time_trans_type_rate = every_work_time_trans_type_rate.groupby('UID')['work_time_trans_type_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, every_work_time_trans_type_rate, on='UID', how='left')\n",
    "\n",
    "# 每天的下午用户该支付方式的count\n",
    "every_afternoon_trans_type_count = transaction_data[(transaction_data.hour >= 12)&(transaction_data.hour <= 17)][['UID', 'day', 'hour', 'trans_type1']]\n",
    "every_afternoon_trans_type_count['every_afternoon_trans_type_count'] = 1\n",
    "every_afternoon_trans_type_count = every_afternoon_trans_type_count.groupby(['UID', 'day', 'hour', 'trans_type1']).agg('count').reset_index()\n",
    "every_afternoon_trans_type_count = every_afternoon_trans_type_count[['UID', 'every_afternoon_trans_type_count']]\n",
    "every_afternoon_trans_type_count = every_afternoon_trans_type_count.groupby('UID')['every_afternoon_trans_type_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_afternoon_trans_type_count.columns = ['UID', 'every_afternoon_trans_type_count_mean', 'every_afternoon_trans_type_count_min', 'every_afternoon_trans_type_count_std', 'every_afternoon_trans_type_count_max']\n",
    "data = pd.merge(data, every_afternoon_trans_type_count, on='UID', how='left')\n",
    "\n",
    "# 每天的下午用户该支付方式的gap\n",
    "every_afternoon_trans_type_gap = transaction_data[(transaction_data.hour >= 12)&(transaction_data.hour <= 17)][['UID', 'day', 'hour', 'trans_type1']]\n",
    "every_afternoon_trans_type_gap['every_afternoon_trans_type_gap'] = 1\n",
    "every_afternoon_trans_type_gap = every_afternoon_trans_type_gap.groupby(['UID', 'day', 'hour', 'trans_type1']).agg('count').reset_index()\n",
    "every_afternoon_trans_type_gap = every_afternoon_trans_type_gap[['UID', 'every_afternoon_trans_type_gap']]\n",
    "every_afternoon_trans_type_gap['before_every_afternoon'] = every_afternoon_trans_type_gap.groupby('UID')['every_afternoon_trans_type_gap'].shift(1)\n",
    "every_afternoon_trans_type_gap['afternoon_trans_type_gap'] = every_afternoon_trans_type_gap['every_afternoon_trans_type_gap'] - every_afternoon_trans_type_gap['before_every_afternoon']\n",
    "every_afternoon_trans_type_gap = every_afternoon_trans_type_gap[['UID', 'afternoon_trans_type_gap']]\n",
    "every_afternoon_trans_type_gap = every_afternoon_trans_type_gap.groupby('UID')['afternoon_trans_type_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_afternoon_trans_type_gap.columns = ['UID', 'every_afternoon_trans_type_gap_sum', 'every_afternoon_trans_type_gap_mean', 'every_afternoon_trans_type_gap_min', 'every_afternoon_trans_type_gap_std', 'every_afternoon_trans_type_gap_max']\n",
    "data = pd.merge(data, every_afternoon_trans_type_gap, on='UID', how='left')\n",
    "\n",
    "# 每天的下午用户该支付方式的rate\n",
    "every_afternoon_trans_type_rate = transaction_data[(transaction_data.hour >= 12)&(transaction_data.hour <= 17)][['UID', 'day', 'hour', 'trans_type1']]\n",
    "every_afternoon_trans_type_rate['every_afternoon_trans_type_rate'] = 1\n",
    "every_afternoon_trans_type_rate = every_afternoon_trans_type_rate.groupby(['UID', 'day', 'hour', 'trans_type1']).agg('count').reset_index()\n",
    "every_afternoon_trans_type_rate = every_afternoon_trans_type_rate[['UID', 'every_afternoon_trans_type_rate']]\n",
    "every_afternoon_trans_type_rate['before_every_afternoon'] = every_afternoon_trans_type_rate.groupby('UID')['every_afternoon_trans_type_rate'].shift(1)\n",
    "every_afternoon_trans_type_rate['afternoon_trans_type_rate'] = every_afternoon_trans_type_rate['every_afternoon_trans_type_rate'] / every_afternoon_trans_type_rate['before_every_afternoon']\n",
    "every_afternoon_trans_type_rate = every_afternoon_trans_type_rate[['UID', 'afternoon_trans_type_rate']]\n",
    "every_afternoon_trans_type_rate = every_afternoon_trans_type_rate.groupby('UID')['afternoon_trans_type_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, every_afternoon_trans_type_rate, on='UID', how='left')\n",
    "\n",
    "# 每天的晚上用户该支付方式的count\n",
    "every_night_trans_type_count = transaction_data[(transaction_data.hour >= 18)&(transaction_data.hour <= 23)][['UID', 'day', 'hour', 'trans_type1']]\n",
    "every_night_trans_type_count['every_night_trans_type_count'] = 1\n",
    "every_night_trans_type_count = every_night_trans_type_count.groupby(['UID', 'day', 'hour', 'trans_type1']).agg('count').reset_index()\n",
    "every_night_trans_type_count = every_night_trans_type_count[['UID', 'every_night_trans_type_count']]\n",
    "every_night_trans_type_count = every_night_trans_type_count.groupby('UID')['every_night_trans_type_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_night_trans_type_count.columns = ['UID', 'every_night_trans_type_count_mean', 'every_night_trans_type_count_min', 'every_night_trans_type_count_std', 'every_night_trans_type_count_max']\n",
    "data = pd.merge(data, every_night_trans_type_count, on='UID', how='left')\n",
    "\n",
    "# 每天的晚上用户该支付方式的gap\n",
    "every_night_trans_type_gap = transaction_data[(transaction_data.hour >= 18)&(transaction_data.hour <= 23)][['UID', 'day', 'hour', 'trans_type1']]\n",
    "every_night_trans_type_gap['every_night_trans_type_gap'] = 1\n",
    "every_night_trans_type_gap = every_night_trans_type_gap.groupby(['UID', 'day', 'hour', 'trans_type1']).agg('count').reset_index()\n",
    "every_night_trans_type_gap = every_night_trans_type_gap[['UID', 'every_night_trans_type_gap']]\n",
    "every_night_trans_type_gap['before_every_night'] = every_night_trans_type_gap.groupby('UID')['every_night_trans_type_gap'].shift(1)\n",
    "every_night_trans_type_gap['night_trans_type_gap'] = every_night_trans_type_gap['every_night_trans_type_gap'] - every_night_trans_type_gap['before_every_night']\n",
    "every_night_trans_type_gap = every_night_trans_type_gap[['UID', 'night_trans_type_gap']]\n",
    "every_night_trans_type_gap = every_night_trans_type_gap.groupby('UID')['night_trans_type_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_night_trans_type_gap.columns = ['UID', 'every_night_trans_type_gap_sum', 'every_night_trans_type_gap_mean', 'every_night_trans_type_gap_min', 'every_night_trans_type_gap_std', 'every_night_trans_type_gap_max']\n",
    "data = pd.merge(data, every_night_trans_type_gap, on='UID', how='left')\n",
    "\n",
    "# 每天的晚上用户该支付方式的rate\n",
    "every_night_trans_type_rate = transaction_data[(transaction_data.hour >= 18)&(transaction_data.hour <= 23)][['UID', 'day', 'hour', 'trans_type1']]\n",
    "every_night_trans_type_rate['every_night_trans_type_rate'] = 1\n",
    "every_night_trans_type_rate = every_night_trans_type_rate.groupby(['UID', 'day', 'hour', 'trans_type1']).agg('count').reset_index()\n",
    "every_night_trans_type_rate = every_night_trans_type_rate[['UID', 'every_night_trans_type_rate']]\n",
    "every_night_trans_type_rate['before_every_night'] = every_night_trans_type_rate.groupby('UID')['every_night_trans_type_rate'].shift(1)\n",
    "every_night_trans_type_rate['night_trans_type_rate'] = every_night_trans_type_rate['every_night_trans_type_rate'] / every_night_trans_type_rate['before_every_night']\n",
    "every_night_trans_type_rate = every_night_trans_type_rate[['UID', 'night_trans_type_rate']]\n",
    "every_night_trans_type_rate = every_night_trans_type_rate.groupby('UID')['night_trans_type_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, every_night_trans_type_rate, on='UID', how='left')\n",
    "\n",
    "# 每三天用户的gap\n",
    "every_three_day_gap = transaction_data[['UID', 'day', 'trans_type1']]\n",
    "every_three_day_gap['every_day_count'] = 1\n",
    "every_three_day_gap = every_three_day_gap.groupby(['UID', 'day', 'trans_type1']).agg('count').reset_index()\n",
    "every_three_day_gap['before_day_count'] = every_three_day_gap.groupby('UID')['every_day_count'].shift(3)\n",
    "every_three_day_gap['every_three_day_gap'] = every_three_day_gap['every_day_count'] - every_three_day_gap['before_day_count']\n",
    "every_three_day_gap = every_three_day_gap[['UID', 'every_three_day_gap']]\n",
    "every_three_day_gap = every_three_day_gap.groupby('UID')['every_three_day_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_three_day_gap.columns = ['UID', 'every_three_day_gap_sum', 'every_three_day_gap_mean', 'every_three_day_gap_min', 'every_three_day_gap_std', 'every_three_day_gap_max']\n",
    "data = pd.merge(data, every_three_day_gap, on='UID', how='left')\n",
    "\n",
    "# 每三天用户的rate\n",
    "every_three_day_rate = transaction_data[['UID', 'day', 'trans_type1']]\n",
    "every_three_day_rate['every_day_count'] = 1\n",
    "every_three_day_rate = every_three_day_rate.groupby(['UID', 'day', 'trans_type1']).agg('count').reset_index()\n",
    "every_three_day_rate['before_day_count'] = every_three_day_rate.groupby('UID')['every_day_count'].shift(3)\n",
    "every_three_day_rate['every_three_day_rate'] = every_three_day_rate['every_day_count'] / every_three_day_rate['before_day_count']\n",
    "every_three_day_rate = every_three_day_rate[['UID', 'every_three_day_rate']]\n",
    "every_three_day_rate = every_three_day_rate.groupby('UID')['every_three_day_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, every_three_day_rate, on='UID', how='left')\n",
    "\n",
    "# 每七天用户的gap\n",
    "every_sixteen_day_gap = transaction_data[['UID', 'day', 'trans_type1']]\n",
    "every_sixteen_day_gap['every_day_count'] = 1\n",
    "every_sixteen_day_gap = every_sixteen_day_gap.groupby(['UID', 'day', 'trans_type1']).agg('count').reset_index()\n",
    "every_sixteen_day_gap['before_day_count'] = every_sixteen_day_gap.groupby('UID')['every_day_count'].shift(7)\n",
    "every_sixteen_day_gap['every_sixteen_day_gap'] = every_sixteen_day_gap['every_day_count'] - every_sixteen_day_gap['before_day_count']\n",
    "every_sixteen_day_gap = every_sixteen_day_gap[['UID', 'every_sixteen_day_gap']]\n",
    "every_sixteen_day_gap = every_sixteen_day_gap.groupby('UID')['every_sixteen_day_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_sixteen_day_gap.columns = ['UID', 'every_sixteen_day_gap_sum', 'every_sixteen_day_gap_mean', 'every_sixteen_day_gap_min', 'every_sixteen_day_gap_std', 'every_sixteen_day_gap_max']\n",
    "data = pd.merge(data, every_sixteen_day_gap, on='UID', how='left')\n",
    "\n",
    "# 每七天用户的rate\n",
    "every_sixteen_day_rate = transaction_data[['UID', 'day', 'trans_type1']]\n",
    "every_sixteen_day_rate['every_day_count'] = 1\n",
    "every_sixteen_day_rate = every_sixteen_day_rate.groupby(['UID', 'day', 'trans_type1']).agg('count').reset_index()\n",
    "every_sixteen_day_rate['before_day_count'] = every_sixteen_day_rate.groupby('UID')['every_day_count'].shift(7)\n",
    "every_sixteen_day_rate['every_sixteen_day_rate'] = every_sixteen_day_rate['every_day_count'] / every_sixteen_day_rate['before_day_count']\n",
    "every_sixteen_day_rate = every_sixteen_day_rate[['UID', 'every_sixteen_day_rate']]\n",
    "every_sixteen_day_rate = every_sixteen_day_rate.groupby('UID')['every_sixteen_day_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, every_sixteen_day_rate, on='UID', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 每一天的每一小时用户支付方式的支付次数\n",
    "market_code_count = transaction_data[['UID', 'day', 'hour', 'market_code']]\n",
    "market_code_count['every_hour_market_code_count'] = 1\n",
    "market_code_count = market_code_count.groupby(['UID', 'day', 'hour', 'market_code']).agg('count').reset_index()\n",
    "market_code_count = market_code_count[['UID', 'every_hour_market_code_count']]\n",
    "market_code_count = market_code_count.groupby('UID')['every_hour_market_code_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "market_code_count.columns = ['UID', 'market_code_count_mean', 'market_code_count_min', 'market_code_count_std', 'market_code_count_max']\n",
    "data = pd.merge(data, market_code_count, on='UID', how='left')\n",
    "\n",
    "# 每一天每一小时用户在该ip用该支付方式支付次数\n",
    "market_code_ip1_count = transaction_data[['UID', 'day', 'hour', 'ip1_sub', 'market_code']]\n",
    "market_code_ip1_count['every_hour_ip1_sub_market_code_count'] = 1\n",
    "market_code_ip1_count = market_code_ip1_count.groupby(['UID', 'day', 'hour', 'ip1_sub', 'market_code']).agg('count').reset_index()\n",
    "market_code_ip1_count = market_code_ip1_count[['UID', 'every_hour_ip1_sub_market_code_count']]\n",
    "market_code_ip1_count = market_code_ip1_count.groupby('UID')['every_hour_ip1_sub_market_code_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "market_code_ip1_count.columns = ['UID', 'market_code_ip1_count_mean', 'market_code_ip1_count_min', 'market_code_ip1_count_std', 'market_code_ip1_count_max']\n",
    "data = pd.merge(data, market_code_ip1_count, on='UID', how='left')\n",
    "\n",
    "# 每一天每一小时用户在该ip用该支付方式支付次数\n",
    "market_code_mac1_count = transaction_data[['UID', 'day', 'hour', 'mac1', 'market_code']]\n",
    "market_code_mac1_count['every_hour_mac1_market_code_count'] = 1\n",
    "market_code_mac1_count = market_code_mac1_count.groupby(['UID', 'day', 'hour', 'mac1', 'market_code']).agg('count').reset_index()\n",
    "market_code_mac1_count = market_code_mac1_count[['UID', 'every_hour_mac1_market_code_count']]\n",
    "market_code_mac1_count = market_code_mac1_count.groupby('UID')['every_hour_mac1_market_code_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "market_code_mac1_count.columns = ['UID', 'market_code_mac1_count_mean', 'market_code_mac1_count_min', 'market_code_mac1_count_std', 'market_code_mac1_count_max']\n",
    "data = pd.merge(data, market_code_mac1_count, on='UID', how='left')\n",
    "\n",
    "# 每一天每一小时用户用device1用该支付方式支付次数\n",
    "trans_type_device_code1_count = transaction_data[['UID', 'day', 'hour', 'device_code1', 'market_code']]\n",
    "trans_type_device_code1_count['every_hour_device_code1_market_code_count'] = 1\n",
    "trans_type_device_code1_count = trans_type_device_code1_count.groupby(['UID', 'day', 'hour', 'device_code1', 'market_code']).agg('count').reset_index()\n",
    "trans_type_device_code1_count = trans_type_device_code1_count[['UID', 'every_hour_device_code1_market_code_count']]\n",
    "trans_type_device_code1_count = trans_type_device_code1_count.groupby('UID')['every_hour_device_code1_market_code_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "trans_type_device_code1_count.columns = ['UID', 'trans_type_device_code1_count_mean', 'trans_type_device_code1_count_min', 'trans_type_device_code1_count_std', 'trans_type_device_code1_count_max']\n",
    "data = pd.merge(data, trans_type_device_code1_count, on='UID', how='left')\n",
    "\n",
    "# 每一天每一小时用户用device2用该支付方式支付次数\n",
    "trans_type_device_code2_count = transaction_data[['UID', 'day', 'hour', 'device_code2', 'market_code']]\n",
    "trans_type_device_code2_count['every_hour_device_code2_market_code_count'] = 1\n",
    "trans_type_device_code2_count = trans_type_device_code2_count.groupby(['UID', 'day', 'hour', 'device_code2', 'market_code']).agg('count').reset_index()\n",
    "trans_type_device_code2_count = trans_type_device_code2_count[['UID', 'every_hour_device_code2_market_code_count']]\n",
    "trans_type_device_code2_count = trans_type_device_code2_count.groupby('UID')['every_hour_device_code2_market_code_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "trans_type_device_code2_count.columns = ['UID', 'trans_type_device_code2_count_mean', 'trans_type_device_code2_count_min', 'trans_type_device_code2_count_std', 'trans_type_device_code2_count_max']\n",
    "data = pd.merge(data, trans_type_device_code2_count, on='UID', how='left')\n",
    "\n",
    "# 每一天每一小时用户用device3用该支付方式支付次数\n",
    "trans_type_device_code3_count = transaction_data[['UID', 'day', 'hour', 'device_code3', 'market_code']]\n",
    "trans_type_device_code3_count['every_hour_device_code3_market_code_count'] = 1\n",
    "trans_type_device_code3_count = trans_type_device_code3_count.groupby(['UID', 'day', 'hour', 'device_code3', 'market_code']).agg('count').reset_index()\n",
    "trans_type_device_code3_count = trans_type_device_code3_count[['UID', 'every_hour_device_code3_market_code_count']]\n",
    "trans_type_device_code3_count = trans_type_device_code3_count.groupby('UID')['every_hour_device_code3_market_code_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "trans_type_device_code3_count.columns = ['UID', 'trans_type_device_code3_count_mean', 'trans_type_device_code3_count_min', 'trans_type_device_code3_count_std', 'trans_type_device_code3_count_max']\n",
    "data = pd.merge(data, trans_type_device_code3_count, on='UID', how='left')\n",
    "\n",
    "# 每一天每一个月用户该支付方式的gap\n",
    "trans_type_hour_gap = transaction_data[['UID', 'day', 'hour', 'market_code']]\n",
    "trans_type_hour_gap['trans_type_hour_gap'] = 1\n",
    "trans_type_hour_gap = trans_type_hour_gap.groupby(['UID', 'day', 'hour', 'market_code']).agg('count').reset_index()\n",
    "trans_type_hour_gap['before_hour_gap'] = trans_type_hour_gap.groupby('UID')['trans_type_hour_gap'].shift(1)\n",
    "trans_type_hour_gap['trans_type_every_hour_gap'] = trans_type_hour_gap['trans_type_hour_gap'] - trans_type_hour_gap['before_hour_gap']\n",
    "trans_type_hour_gap = trans_type_hour_gap[['UID', 'trans_type_every_hour_gap']]\n",
    "trans_type_hour_gap = trans_type_hour_gap.groupby('UID')['trans_type_every_hour_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "trans_type_hour_gap.columns = ['UID', 'trans_type_hour_gap_sum', 'trans_type_hour_gap_mean', 'trans_type_hour_gap_min', 'trans_type_hour_gap_std', 'trans_type_hour_gap_max']\n",
    "data = pd.merge(data, trans_type_hour_gap, on='UID', how='left')\n",
    "\n",
    "# 每一天每一个月用户该支付方式的rate\n",
    "trans_type_hour_rate = transaction_data[['UID', 'day', 'hour', 'market_code']]\n",
    "trans_type_hour_rate['trans_type_hour_rate'] = 1\n",
    "trans_type_hour_rate = trans_type_hour_rate.groupby(['UID', 'day', 'hour', 'market_code']).agg('count').reset_index()\n",
    "trans_type_hour_rate['before_hour_rate'] = trans_type_hour_rate.groupby('UID')['trans_type_hour_rate'].shift(1)\n",
    "trans_type_hour_rate['trans_type_every_hour_rate'] = trans_type_hour_rate['trans_type_hour_rate'] / trans_type_hour_rate['before_hour_rate']\n",
    "trans_type_hour_rate = trans_type_hour_rate[['UID', 'trans_type_every_hour_rate']]\n",
    "trans_type_hour_rate = trans_type_hour_rate.groupby('UID')['trans_type_every_hour_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, trans_type_hour_rate, on='UID', how='left')\n",
    "\n",
    "\n",
    "# 每一天的用户支付方式的支付次数\n",
    "market_code_count = transaction_data[['UID', 'day', 'market_code']]\n",
    "market_code_count['every_day_market_code_count'] = 1\n",
    "market_code_count = market_code_count.groupby(['UID', 'day', 'market_code']).agg('count').reset_index()\n",
    "market_code_count = market_code_count[['UID', 'every_day_market_code_count']]\n",
    "market_code_count = market_code_count.groupby('UID')['every_day_market_code_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "market_code_count.columns = ['UID', 'market_code_count_mean_day', 'market_code_count_min_day', 'market_code_count_std_day', 'market_code_count_max_day']\n",
    "data = pd.merge(data, market_code_count, on='UID', how='left')\n",
    "\n",
    "# 每一天用户在该ip用该支付方式支付次数\n",
    "market_code_ip1_count = transaction_data[['UID', 'day', 'ip1_sub', 'market_code']]\n",
    "market_code_ip1_count['every_day_ip1_sub_market_code_count'] = 1\n",
    "market_code_ip1_count = market_code_ip1_count.groupby(['UID', 'day', 'ip1_sub', 'market_code']).agg('count').reset_index()\n",
    "market_code_ip1_count = market_code_ip1_count[['UID', 'every_day_ip1_sub_market_code_count']]\n",
    "market_code_ip1_count = market_code_ip1_count.groupby('UID')['every_day_ip1_sub_market_code_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "market_code_ip1_count.columns = ['UID', 'market_code_ip1_count_mean_day', 'market_code_ip1_count_min_day', 'market_code_ip1_count_std_day', 'market_code_ip1_count_max_day']\n",
    "data = pd.merge(data, market_code_ip1_count, on='UID', how='left')\n",
    "\n",
    "# 每一天用户在该mac用该支付方式支付次数\n",
    "market_code_mac1_count = transaction_data[['UID', 'day', 'mac1', 'market_code']]\n",
    "market_code_mac1_count['every_day_mac1_market_code_count'] = 1\n",
    "market_code_mac1_count = market_code_mac1_count.groupby(['UID', 'day', 'mac1', 'market_code']).agg('count').reset_index()\n",
    "market_code_mac1_count = market_code_mac1_count[['UID', 'every_day_mac1_market_code_count']]\n",
    "market_code_mac1_count = market_code_mac1_count.groupby('UID')['every_day_mac1_market_code_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "market_code_mac1_count.columns = ['UID', 'market_code_mac1_count_mean_day', 'market_code_mac1_count_min_day', 'market_code_mac1_count_std_day', 'market_code_mac1_count_max_day']\n",
    "data = pd.merge(data, market_code_mac1_count, on='UID', how='left')\n",
    "\n",
    "# 每一天用户用device1用该支付方式支付次数\n",
    "trans_type_device_code1_count = transaction_data[['UID', 'day', 'device_code1', 'market_code']]\n",
    "trans_type_device_code1_count['every_day_device_code1_market_code_count'] = 1\n",
    "trans_type_device_code1_count = trans_type_device_code1_count.groupby(['UID', 'day', 'device_code1', 'market_code']).agg('count').reset_index()\n",
    "trans_type_device_code1_count = trans_type_device_code1_count[['UID', 'every_day_device_code1_market_code_count']]\n",
    "trans_type_device_code1_count = trans_type_device_code1_count.groupby('UID')['every_day_device_code1_market_code_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "trans_type_device_code1_count.columns = ['UID', 'trans_type_device_code1_count_mean_day', 'trans_type_device_code1_count_min_day', 'trans_type_device_code1_count_std_day', 'trans_type_device_code1_count_max_day']\n",
    "data = pd.merge(data, trans_type_device_code1_count, on='UID', how='left')\n",
    "\n",
    "# 每一天用户用device2用该支付方式支付次数\n",
    "trans_type_device_code2_count = transaction_data[['UID', 'day', 'device_code2', 'market_code']]\n",
    "trans_type_device_code2_count['every_day_device_code2_market_code_count'] = 1\n",
    "trans_type_device_code2_count = trans_type_device_code2_count.groupby(['UID', 'day', 'device_code2', 'market_code']).agg('count').reset_index()\n",
    "trans_type_device_code2_count = trans_type_device_code2_count[['UID', 'every_day_device_code2_market_code_count']]\n",
    "trans_type_device_code2_count = trans_type_device_code2_count.groupby('UID')['every_day_device_code2_market_code_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "trans_type_device_code2_count.columns = ['UID', 'trans_type_device_code2_count_mean_day', 'trans_type_device_code2_count_min_day', 'trans_type_device_code2_count_std_day', 'trans_type_device_code2_count_max_day']\n",
    "data = pd.merge(data, trans_type_device_code2_count, on='UID', how='left')\n",
    "\n",
    "# 每一天用户用device3用该支付方式支付次数\n",
    "trans_type_device_code3_count = transaction_data[['UID', 'day', 'device_code3', 'market_code']]\n",
    "trans_type_device_code3_count['every_day_device_code3_market_code_count'] = 1\n",
    "trans_type_device_code3_count = trans_type_device_code3_count.groupby(['UID', 'day', 'device_code3', 'market_code']).agg('count').reset_index()\n",
    "trans_type_device_code3_count = trans_type_device_code3_count[['UID', 'every_day_device_code3_market_code_count']]\n",
    "trans_type_device_code3_count = trans_type_device_code3_count.groupby('UID')['every_day_device_code3_market_code_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "trans_type_device_code3_count.columns = ['UID', 'trans_type_device_code3_count_mean_day', 'trans_type_device_code3_count_min_day', 'trans_type_device_code3_count_std_day', 'trans_type_device_code3_count_max_day']\n",
    "data = pd.merge(data, trans_type_device_code3_count, on='UID', how='left')\n",
    "\n",
    "# 每一天每一个月用户该支付方式的gap\n",
    "trans_type_day_gap = transaction_data[['UID', 'day', 'market_code']]\n",
    "trans_type_day_gap['trans_type_day_gap'] = 1\n",
    "trans_type_day_gap = trans_type_day_gap.groupby(['UID', 'day', 'market_code']).agg('count').reset_index()\n",
    "trans_type_day_gap['before_day_gap'] = trans_type_day_gap.groupby('UID')['trans_type_day_gap'].shift(1)\n",
    "trans_type_day_gap['trans_type_every_day_gap'] = trans_type_day_gap['trans_type_day_gap'] - trans_type_day_gap['before_day_gap']\n",
    "trans_type_day_gap = trans_type_day_gap[['UID', 'trans_type_every_day_gap']]\n",
    "trans_type_day_gap = trans_type_day_gap.groupby('UID')['trans_type_every_day_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "trans_type_day_gap.columns = ['UID', 'trans_type_day_gap_sum_day', 'trans_type_day_gap_mean_day', 'trans_type_day_gap_min_day', 'trans_type_day_gap_std_day', 'trans_type_day_gap_max_day']\n",
    "data = pd.merge(data, trans_type_day_gap, on='UID', how='left')\n",
    "\n",
    "# 每一天每一个月用户该支付方式的rate\n",
    "trans_type_day_rate = transaction_data[['UID', 'day', 'market_code']]\n",
    "trans_type_day_rate['trans_type_day_rate'] = 1\n",
    "trans_type_day_rate = trans_type_day_rate.groupby(['UID', 'day', 'market_code']).agg('count').reset_index()\n",
    "trans_type_day_rate['before_day_rate'] = trans_type_day_rate.groupby('UID')['trans_type_day_rate'].shift(1)\n",
    "trans_type_day_rate['trans_type_every_day_rate'] = trans_type_day_rate['trans_type_day_rate'] / trans_type_day_rate['before_day_rate']\n",
    "trans_type_day_rate = trans_type_day_rate[['UID', 'trans_type_every_day_rate']]\n",
    "trans_type_day_rate = trans_type_day_rate.groupby('UID')['trans_type_every_day_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, trans_type_day_rate, on='UID', how='left')\n",
    "\n",
    "\n",
    "# 每天的半夜用户该支付方式的count\n",
    "every_morning_trans_type_count = transaction_data[transaction_data.hour <= 5][['UID', 'day', 'hour', 'market_code']]\n",
    "every_morning_trans_type_count['every_morning_trans_type_count'] = 1\n",
    "every_morning_trans_type_count = every_morning_trans_type_count.groupby(['UID', 'day', 'hour', 'market_code']).agg('count').reset_index()\n",
    "every_morning_trans_type_count = every_morning_trans_type_count[['UID', 'every_morning_trans_type_count']]\n",
    "every_morning_trans_type_count = every_morning_trans_type_count.groupby('UID')['every_morning_trans_type_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_morning_trans_type_count.columns = ['UID', 'every_morning_trans_type_count_mean', 'every_morning_trans_type_count_min', 'every_morning_trans_type_count_std', 'every_morning_trans_type_count_max']\n",
    "data = pd.merge(data, every_morning_trans_type_count, on='UID', how='left')\n",
    "\n",
    "# 每天的半夜用户该支付方式的gap\n",
    "every_morning_trans_type_gap = transaction_data[transaction_data.hour <= 5][['UID', 'day', 'hour', 'market_code']]\n",
    "every_morning_trans_type_gap['every_morning_trans_type_gap'] = 1\n",
    "every_morning_trans_type_gap = every_morning_trans_type_gap.groupby(['UID', 'day', 'hour', 'market_code']).agg('count').reset_index()\n",
    "every_morning_trans_type_gap = every_morning_trans_type_gap[['UID', 'every_morning_trans_type_gap']]\n",
    "every_morning_trans_type_gap['before_every_morning'] = every_morning_trans_type_gap.groupby('UID')['every_morning_trans_type_gap'].shift(1)\n",
    "every_morning_trans_type_gap['morning_trans_type_gap'] = every_morning_trans_type_gap['every_morning_trans_type_gap'] - every_morning_trans_type_gap['before_every_morning']\n",
    "every_morning_trans_type_gap = every_morning_trans_type_gap[['UID', 'morning_trans_type_gap']]\n",
    "every_morning_trans_type_gap = every_morning_trans_type_gap.groupby('UID')['morning_trans_type_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_morning_trans_type_gap.columns = ['UID', 'every_morning_trans_type_gap_sum', 'every_morning_trans_type_gap_mean', 'every_morning_trans_type_gap_min', 'every_morning_trans_type_gap_std', 'every_morning_trans_type_gap_max']\n",
    "data = pd.merge(data, every_morning_trans_type_gap, on='UID', how='left')\n",
    "\n",
    "# 每天的半夜用户该支付方式的rate\n",
    "every_morning_trans_type_rate = transaction_data[transaction_data.hour <= 5][['UID', 'day', 'hour', 'market_code']]\n",
    "every_morning_trans_type_rate['every_morning_trans_type_rate'] = 1\n",
    "every_morning_trans_type_rate = every_morning_trans_type_rate.groupby(['UID', 'day', 'hour', 'market_code']).agg('count').reset_index()\n",
    "every_morning_trans_type_rate = every_morning_trans_type_rate[['UID', 'every_morning_trans_type_rate']]\n",
    "every_morning_trans_type_rate['before_every_morning'] = every_morning_trans_type_rate.groupby('UID')['every_morning_trans_type_rate'].shift(1)\n",
    "every_morning_trans_type_rate['morning_trans_type_rate'] = every_morning_trans_type_rate['every_morning_trans_type_rate'] / every_morning_trans_type_rate['before_every_morning']\n",
    "every_morning_trans_type_rate = every_morning_trans_type_rate[['UID', 'morning_trans_type_rate']]\n",
    "every_morning_trans_type_rate = every_morning_trans_type_rate.groupby('UID')['morning_trans_type_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, every_morning_trans_type_rate, on='UID', how='left')\n",
    "\n",
    "# 每天的早上用户该支付方式的count\n",
    "every_work_time_trans_type_count = transaction_data[(transaction_data.hour >= 6)&(transaction_data.hour <= 11)][['UID', 'day', 'hour', 'market_code']]\n",
    "every_work_time_trans_type_count['every_work_time_trans_type_count'] = 1\n",
    "every_work_time_trans_type_count = every_work_time_trans_type_count.groupby(['UID', 'day', 'hour', 'market_code']).agg('count').reset_index()\n",
    "every_work_time_trans_type_count = every_work_time_trans_type_count[['UID', 'every_work_time_trans_type_count']]\n",
    "every_work_time_trans_type_count = every_work_time_trans_type_count.groupby('UID')['every_work_time_trans_type_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_work_time_trans_type_count.columns = ['UID', 'every_work_time_trans_type_count_mean', 'every_work_time_trans_type_count_min', 'every_work_time_trans_type_count_std', 'every_work_time_trans_type_count_max']\n",
    "data = pd.merge(data, every_work_time_trans_type_count, on='UID', how='left')\n",
    "\n",
    "# 每天的早上用户该支付方式的gap\n",
    "every_work_time_trans_type_gap = transaction_data[(transaction_data.hour >= 6)&(transaction_data.hour <= 11)][['UID', 'day', 'hour', 'market_code']]\n",
    "every_work_time_trans_type_gap['every_work_time_trans_type_gap'] = 1\n",
    "every_work_time_trans_type_gap = every_work_time_trans_type_gap.groupby(['UID', 'day', 'hour', 'market_code']).agg('count').reset_index()\n",
    "every_work_time_trans_type_gap = every_work_time_trans_type_gap[['UID', 'every_work_time_trans_type_gap']]\n",
    "every_work_time_trans_type_gap['before_every_work_time'] = every_work_time_trans_type_gap.groupby('UID')['every_work_time_trans_type_gap'].shift(1)\n",
    "every_work_time_trans_type_gap['work_time_trans_type_gap'] = every_work_time_trans_type_gap['every_work_time_trans_type_gap'] - every_work_time_trans_type_gap['before_every_work_time']\n",
    "every_work_time_trans_type_gap = every_work_time_trans_type_gap[['UID', 'work_time_trans_type_gap']]\n",
    "every_work_time_trans_type_gap = every_work_time_trans_type_gap.groupby('UID')['work_time_trans_type_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_work_time_trans_type_gap.columns = ['UID', 'every_work_time_trans_type_gap_sum', 'every_work_time_trans_type_gap_mean', 'every_work_time_trans_type_gap_min', 'every_work_time_trans_type_gap_std', 'every_work_time_trans_type_gap_max']\n",
    "data = pd.merge(data, every_work_time_trans_type_gap, on='UID', how='left')\n",
    "\n",
    "# 每天的早上用户该支付方式的rate\n",
    "every_work_time_trans_type_rate = transaction_data[(transaction_data.hour >= 6)&(transaction_data.hour <= 11)][['UID', 'day', 'hour', 'market_code']]\n",
    "every_work_time_trans_type_rate['every_work_time_trans_type_rate'] = 1\n",
    "every_work_time_trans_type_rate = every_work_time_trans_type_rate.groupby(['UID', 'day', 'hour', 'market_code']).agg('count').reset_index()\n",
    "every_work_time_trans_type_rate = every_work_time_trans_type_rate[['UID', 'every_work_time_trans_type_rate']]\n",
    "every_work_time_trans_type_rate['before_every_work_time'] = every_work_time_trans_type_rate.groupby('UID')['every_work_time_trans_type_rate'].shift(1)\n",
    "every_work_time_trans_type_rate['work_time_trans_type_rate'] = every_work_time_trans_type_rate['every_work_time_trans_type_rate'] / every_work_time_trans_type_rate['before_every_work_time']\n",
    "every_work_time_trans_type_rate = every_work_time_trans_type_rate[['UID', 'work_time_trans_type_rate']]\n",
    "every_work_time_trans_type_rate = every_work_time_trans_type_rate.groupby('UID')['work_time_trans_type_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, every_work_time_trans_type_rate, on='UID', how='left')\n",
    "\n",
    "# 每天的下午用户该支付方式的count\n",
    "every_afternoon_trans_type_count = transaction_data[(transaction_data.hour >= 12)&(transaction_data.hour <= 17)][['UID', 'day', 'hour', 'market_code']]\n",
    "every_afternoon_trans_type_count['every_afternoon_trans_type_count'] = 1\n",
    "every_afternoon_trans_type_count = every_afternoon_trans_type_count.groupby(['UID', 'day', 'hour', 'market_code']).agg('count').reset_index()\n",
    "every_afternoon_trans_type_count = every_afternoon_trans_type_count[['UID', 'every_afternoon_trans_type_count']]\n",
    "every_afternoon_trans_type_count = every_afternoon_trans_type_count.groupby('UID')['every_afternoon_trans_type_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_afternoon_trans_type_count.columns = ['UID', 'every_afternoon_trans_type_count_mean', 'every_afternoon_trans_type_count_min', 'every_afternoon_trans_type_count_std', 'every_afternoon_trans_type_count_max']\n",
    "data = pd.merge(data, every_afternoon_trans_type_count, on='UID', how='left')\n",
    "\n",
    "# 每天的下午用户该支付方式的gap\n",
    "every_afternoon_trans_type_gap = transaction_data[(transaction_data.hour >= 12)&(transaction_data.hour <= 17)][['UID', 'day', 'hour', 'market_code']]\n",
    "every_afternoon_trans_type_gap['every_afternoon_trans_type_gap'] = 1\n",
    "every_afternoon_trans_type_gap = every_afternoon_trans_type_gap.groupby(['UID', 'day', 'hour', 'market_code']).agg('count').reset_index()\n",
    "every_afternoon_trans_type_gap = every_afternoon_trans_type_gap[['UID', 'every_afternoon_trans_type_gap']]\n",
    "every_afternoon_trans_type_gap['before_every_afternoon'] = every_afternoon_trans_type_gap.groupby('UID')['every_afternoon_trans_type_gap'].shift(1)\n",
    "every_afternoon_trans_type_gap['afternoon_trans_type_gap'] = every_afternoon_trans_type_gap['every_afternoon_trans_type_gap'] - every_afternoon_trans_type_gap['before_every_afternoon']\n",
    "every_afternoon_trans_type_gap = every_afternoon_trans_type_gap[['UID', 'afternoon_trans_type_gap']]\n",
    "every_afternoon_trans_type_gap = every_afternoon_trans_type_gap.groupby('UID')['afternoon_trans_type_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_afternoon_trans_type_gap.columns = ['UID', 'every_afternoon_trans_type_gap_sum', 'every_afternoon_trans_type_gap_mean', 'every_afternoon_trans_type_gap_min', 'every_afternoon_trans_type_gap_std', 'every_afternoon_trans_type_gap_max']\n",
    "data = pd.merge(data, every_afternoon_trans_type_gap, on='UID', how='left')\n",
    "\n",
    "# 每天的下午用户该支付方式的rate\n",
    "every_afternoon_trans_type_rate = transaction_data[(transaction_data.hour >= 12)&(transaction_data.hour <= 17)][['UID', 'day', 'hour', 'market_code']]\n",
    "every_afternoon_trans_type_rate['every_afternoon_trans_type_rate'] = 1\n",
    "every_afternoon_trans_type_rate = every_afternoon_trans_type_rate.groupby(['UID', 'day', 'hour', 'market_code']).agg('count').reset_index()\n",
    "every_afternoon_trans_type_rate = every_afternoon_trans_type_rate[['UID', 'every_afternoon_trans_type_rate']]\n",
    "every_afternoon_trans_type_rate['before_every_afternoon'] = every_afternoon_trans_type_rate.groupby('UID')['every_afternoon_trans_type_rate'].shift(1)\n",
    "every_afternoon_trans_type_rate['afternoon_trans_type_rate'] = every_afternoon_trans_type_rate['every_afternoon_trans_type_rate'] / every_afternoon_trans_type_rate['before_every_afternoon']\n",
    "every_afternoon_trans_type_rate = every_afternoon_trans_type_rate[['UID', 'afternoon_trans_type_rate']]\n",
    "every_afternoon_trans_type_rate = every_afternoon_trans_type_rate.groupby('UID')['afternoon_trans_type_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, every_afternoon_trans_type_rate, on='UID', how='left')\n",
    "\n",
    "# 每天的晚上用户该支付方式的count\n",
    "every_night_trans_type_count = transaction_data[(transaction_data.hour >= 18)&(transaction_data.hour <= 23)][['UID', 'day', 'hour', 'market_code']]\n",
    "every_night_trans_type_count['every_night_trans_type_count'] = 1\n",
    "every_night_trans_type_count = every_night_trans_type_count.groupby(['UID', 'day', 'hour', 'market_code']).agg('count').reset_index()\n",
    "every_night_trans_type_count = every_night_trans_type_count[['UID', 'every_night_trans_type_count']]\n",
    "every_night_trans_type_count = every_night_trans_type_count.groupby('UID')['every_night_trans_type_count'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_night_trans_type_count.columns = ['UID', 'every_night_trans_type_count_mean', 'every_night_trans_type_count_min', 'every_night_trans_type_count_std', 'every_night_trans_type_count_max']\n",
    "data = pd.merge(data, every_night_trans_type_count, on='UID', how='left')\n",
    "\n",
    "# 每天的晚上用户该支付方式的gap\n",
    "every_night_trans_type_gap = transaction_data[(transaction_data.hour >= 18)&(transaction_data.hour <= 23)][['UID', 'day', 'hour', 'market_code']]\n",
    "every_night_trans_type_gap['every_night_trans_type_gap'] = 1\n",
    "every_night_trans_type_gap = every_night_trans_type_gap.groupby(['UID', 'day', 'hour', 'market_code']).agg('count').reset_index()\n",
    "every_night_trans_type_gap = every_night_trans_type_gap[['UID', 'every_night_trans_type_gap']]\n",
    "every_night_trans_type_gap['before_every_night'] = every_night_trans_type_gap.groupby('UID')['every_night_trans_type_gap'].shift(1)\n",
    "every_night_trans_type_gap['night_trans_type_gap'] = every_night_trans_type_gap['every_night_trans_type_gap'] - every_night_trans_type_gap['before_every_night']\n",
    "every_night_trans_type_gap = every_night_trans_type_gap[['UID', 'night_trans_type_gap']]\n",
    "every_night_trans_type_gap = every_night_trans_type_gap.groupby('UID')['night_trans_type_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_night_trans_type_gap.columns = ['UID', 'every_night_trans_type_gap_sum', 'every_night_trans_type_gap_mean', 'every_night_trans_type_gap_min', 'every_night_trans_type_gap_std', 'every_night_trans_type_gap_max']\n",
    "data = pd.merge(data, every_night_trans_type_gap, on='UID', how='left')\n",
    "\n",
    "# 每天的晚上用户该支付方式的rate\n",
    "every_night_trans_type_rate = transaction_data[(transaction_data.hour >= 18)&(transaction_data.hour <= 23)][['UID', 'day', 'hour', 'market_code']]\n",
    "every_night_trans_type_rate['every_night_trans_type_rate'] = 1\n",
    "every_night_trans_type_rate = every_night_trans_type_rate.groupby(['UID', 'day', 'hour', 'market_code']).agg('count').reset_index()\n",
    "every_night_trans_type_rate = every_night_trans_type_rate[['UID', 'every_night_trans_type_rate']]\n",
    "every_night_trans_type_rate['before_every_night'] = every_night_trans_type_rate.groupby('UID')['every_night_trans_type_rate'].shift(1)\n",
    "every_night_trans_type_rate['night_trans_type_rate'] = every_night_trans_type_rate['every_night_trans_type_rate'] / every_night_trans_type_rate['before_every_night']\n",
    "every_night_trans_type_rate = every_night_trans_type_rate[['UID', 'night_trans_type_rate']]\n",
    "every_night_trans_type_rate = every_night_trans_type_rate.groupby('UID')['night_trans_type_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, every_night_trans_type_rate, on='UID', how='left')\n",
    "\n",
    "# 每三天用户的gap\n",
    "every_three_day_gap = transaction_data[['UID', 'day', 'market_code']]\n",
    "every_three_day_gap['every_day_count'] = 1\n",
    "every_three_day_gap = every_three_day_gap.groupby(['UID', 'day', 'market_code']).agg('count').reset_index()\n",
    "every_three_day_gap['before_day_count'] = every_three_day_gap.groupby('UID')['every_day_count'].shift(3)\n",
    "every_three_day_gap['every_three_day_gap'] = every_three_day_gap['every_day_count'] - every_three_day_gap['before_day_count']\n",
    "every_three_day_gap = every_three_day_gap[['UID', 'every_three_day_gap']]\n",
    "every_three_day_gap = every_three_day_gap.groupby('UID')['every_three_day_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_three_day_gap.columns = ['UID', 'every_three_day_gap_sum', 'every_three_day_gap_mean', 'every_three_day_gap_min', 'every_three_day_gap_std', 'every_three_day_gap_max']\n",
    "data = pd.merge(data, every_three_day_gap, on='UID', how='left')\n",
    "\n",
    "# 每三天用户的rate\n",
    "every_three_day_rate = transaction_data[['UID', 'day', 'market_code']]\n",
    "every_three_day_rate['every_day_count'] = 1\n",
    "every_three_day_rate = every_three_day_rate.groupby(['UID', 'day', 'market_code']).agg('count').reset_index()\n",
    "every_three_day_rate['before_day_count'] = every_three_day_rate.groupby('UID')['every_day_count'].shift(3)\n",
    "every_three_day_rate['every_three_day_rate'] = every_three_day_rate['every_day_count'] / every_three_day_rate['before_day_count']\n",
    "every_three_day_rate = every_three_day_rate[['UID', 'every_three_day_rate']]\n",
    "every_three_day_rate = every_three_day_rate.groupby('UID')['every_three_day_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, every_three_day_rate, on='UID', how='left')\n",
    "\n",
    "# 每七天用户的gap\n",
    "every_sixteen_day_gap = transaction_data[['UID', 'day', 'market_code']]\n",
    "every_sixteen_day_gap['every_day_count'] = 1\n",
    "every_sixteen_day_gap = every_sixteen_day_gap.groupby(['UID', 'day', 'market_code']).agg('count').reset_index()\n",
    "every_sixteen_day_gap['before_day_count'] = every_sixteen_day_gap.groupby('UID')['every_day_count'].shift(7)\n",
    "every_sixteen_day_gap['every_sixteen_day_gap'] = every_sixteen_day_gap['every_day_count'] - every_sixteen_day_gap['before_day_count']\n",
    "every_sixteen_day_gap = every_sixteen_day_gap[['UID', 'every_sixteen_day_gap']]\n",
    "every_sixteen_day_gap = every_sixteen_day_gap.groupby('UID')['every_sixteen_day_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_sixteen_day_gap.columns = ['UID', 'every_sixteen_day_gap_sum', 'every_sixteen_day_gap_mean', 'every_sixteen_day_gap_min', 'every_sixteen_day_gap_std', 'every_sixteen_day_gap_max']\n",
    "data = pd.merge(data, every_sixteen_day_gap, on='UID', how='left')\n",
    "\n",
    "# 每七天用户的rate\n",
    "every_sixteen_day_rate = transaction_data[['UID', 'day', 'market_code']]\n",
    "every_sixteen_day_rate['every_day_count'] = 1\n",
    "every_sixteen_day_rate = every_sixteen_day_rate.groupby(['UID', 'day', 'market_code']).agg('count').reset_index()\n",
    "every_sixteen_day_rate['before_day_count'] = every_sixteen_day_rate.groupby('UID')['every_day_count'].shift(7)\n",
    "every_sixteen_day_rate['every_sixteen_day_rate'] = every_sixteen_day_rate['every_day_count'] / every_sixteen_day_rate['before_day_count']\n",
    "every_sixteen_day_rate = every_sixteen_day_rate[['UID', 'every_sixteen_day_rate']]\n",
    "every_sixteen_day_rate = every_sixteen_day_rate.groupby('UID')['every_sixteen_day_rate'].agg('sum').reset_index()\n",
    "data = pd.merge(data, every_sixteen_day_rate, on='UID', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "时间差特征\n",
    "\"\"\"\n",
    "# 每一天用户操作的时间差\n",
    "every_day_time_gap = operation_data[['UID', 'day']]\n",
    "every_day_time_gap = every_day_time_gap.sort_values(['UID', 'day'])\n",
    "every_day_time_gap['last_day'] = every_day_time_gap.groupby('UID')['day'].shift(1)\n",
    "every_day_time_gap['last_day_gap'] = every_day_time_gap['day'] - every_day_time_gap['last_day']\n",
    "every_day_time_gap = every_day_time_gap[['UID', 'last_day_gap']]\n",
    "every_day_time_gap = every_day_time_gap.groupby('UID')['last_day_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_day_time_gap.columns = ['UID', 'every_day_time_gap_sum', 'every_day_time_gap_std', 'every_day_time_gap_min', 'every_day_time_gap_mean', 'every_day_time_gap_max']\n",
    "data = pd.merge(data, every_day_time_gap, on='UID', how='left')\n",
    "\n",
    "# 每一天每一小时用户操作的时间差\n",
    "every_hour_time_gap = operation_data[['UID', 'day', 'hour']]\n",
    "every_hour_time_gap['hour_'] = every_hour_time_gap[['day', 'hour']].apply(lambda x: 24 * (x['day'] - 1) + x['hour'], axis=1)\n",
    "every_hour_time_gap = every_hour_time_gap[['UID', 'hour_']]\n",
    "every_hour_time_gap = every_hour_time_gap.sort_values(['UID', 'hour_'])\n",
    "every_hour_time_gap['last_day_hour'] = every_hour_time_gap.groupby('UID')['hour_'].shift(1)\n",
    "every_hour_time_gap['every_hour_time_gap'] = every_hour_time_gap['hour_'] - every_hour_time_gap['last_day_hour']\n",
    "every_hour_time_gap = every_hour_time_gap[['UID', 'every_hour_time_gap']]\n",
    "every_hour_time_gap = every_hour_time_gap.groupby('UID')['every_hour_time_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_hour_time_gap.columns = ['UID', 'every_hour_time_gap_sum', 'every_hour_time_gap_std', 'every_hour_time_gap_mean', 'every_hour_time_gap_min', 'every_hour_time_gap_max']\n",
    "data = pd.merge(data, every_hour_time_gap, on='UID', how='left')\n",
    "\n",
    "# 每一天每一分钟用户操作的时间差\n",
    "every_minute_time_gap = operation_data[['UID', 'day', 'time']]\n",
    "every_minute_time_gap['minute'] = every_minute_time_gap[['day', 'time']].apply(lambda x: 1440 * (x['day'] - 1) + 60 * int(x['time'].split(':')[0]) + int(x['time'].split(':')[1]), axis=1)\n",
    "every_minute_time_gap = every_minute_time_gap[['UID', 'minute']]\n",
    "every_minute_time_gap = every_minute_time_gap.sort_values(['UID', 'minute'])\n",
    "every_minute_time_gap['last_minute'] = every_minute_time_gap.groupby('UID')['minute'].shift(1)\n",
    "every_minute_time_gap['last_minute_gap'] = every_minute_time_gap['minute'] - every_minute_time_gap['last_minute']\n",
    "every_minute_time_gap = every_minute_time_gap[['UID', 'last_minute_gap']]\n",
    "every_minute_time_gap = every_minute_time_gap.groupby('UID')['last_minute_gap'].agg({'sum', 'mean', 'std', 'max', 'min'}).reset_index()\n",
    "data = pd.merge(data, every_minute_time_gap, on='UID', how='left')\n",
    "\n",
    "# 每三天用户操作的时间差\n",
    "every_three_day_time_gap = operation_data[['UID', 'day']]\n",
    "every_three_day_time_gap = every_three_day_time_gap.sort_values(['UID', 'day'])\n",
    "every_three_day_time_gap['last_three_day'] = every_three_day_time_gap.groupby('UID')['day'].shift(3)\n",
    "every_three_day_time_gap['last_three_day_gap'] = every_three_day_time_gap['day'] - every_three_day_time_gap['last_three_day']\n",
    "every_three_day_time_gap = every_three_day_time_gap[['UID', 'last_three_day_gap']]\n",
    "every_three_day_time_gap = every_three_day_time_gap.groupby('UID')['last_three_day_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_three_day_time_gap.columns = ['UID', 'every_three_day_time_gap_sum', 'every_three_day_time_gap_std', 'every_three_day_time_gap_min', 'every_three_day_time_gap_mean', 'every_three_day_time_gap_max']\n",
    "data = pd.merge(data, every_three_day_time_gap, on='UID', how='left')\n",
    "\n",
    "# 每七天用户操作的时间差\n",
    "every_sixteen_day_time_gap = operation_data[['UID', 'day']]\n",
    "every_sixteen_day_time_gap = every_sixteen_day_time_gap.sort_values(['UID', 'day'])\n",
    "every_sixteen_day_time_gap['last_sixteen_day'] = every_sixteen_day_time_gap.groupby('UID')['day'].shift(7)\n",
    "every_sixteen_day_time_gap['last_sixteen_day_gap'] = every_sixteen_day_time_gap['day'] - every_sixteen_day_time_gap['last_sixteen_day']\n",
    "every_sixteen_day_time_gap = every_sixteen_day_time_gap[['UID', 'last_sixteen_day_gap']]\n",
    "every_sixteen_day_time_gap = every_sixteen_day_time_gap.groupby('UID')['last_sixteen_day_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_sixteen_day_time_gap.columns = ['UID', 'every_sixteen_day_time_gap_sum', 'every_sixteen_day_time_gap_std', 'every_sixteen_day_time_gap_min', 'every_sixteen_day_time_gap_mean', 'every_sixteen_day_time_gap_max']\n",
    "data = pd.merge(data, every_sixteen_day_time_gap, on='UID', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "时间差特征\n",
    "\"\"\"\n",
    "# 每一天用户操作的时间差\n",
    "every_day_time2_gap = transaction_data[['UID', 'day']]\n",
    "every_day_time2_gap = every_day_time2_gap.sort_values(['UID', 'day'])\n",
    "every_day_time2_gap['last_day'] = every_day_time2_gap.groupby('UID')['day'].shift(1)\n",
    "every_day_time2_gap['last_day_gap'] = every_day_time2_gap['day'] - every_day_time2_gap['last_day']\n",
    "every_day_time2_gap = every_day_time2_gap[['UID', 'last_day_gap']]\n",
    "every_day_time2_gap = every_day_time2_gap.groupby('UID')['last_day_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_day_time2_gap.columns = ['UID', 'every_day_time2_gap_sum', 'every_day_time2_gap_std', 'every_day_time2_gap_min', 'every_day_time2_gap_mean', 'every_day_time2_gap_max']\n",
    "data = pd.merge(data, every_day_time2_gap, on='UID', how='left')\n",
    "\n",
    "# 每一天每一小时用户操作的时间差\n",
    "every_hour_time2_gap = transaction_data[['UID', 'day', 'hour']]\n",
    "every_hour_time2_gap['hour_'] = every_hour_time2_gap[['day', 'hour']].apply(lambda x: 24 * (x['day'] - 1) + x['hour'], axis=1)\n",
    "every_hour_time2_gap = every_hour_time2_gap[['UID', 'hour_']]\n",
    "every_hour_time2_gap = every_hour_time2_gap.sort_values(['UID', 'hour_'])\n",
    "every_hour_time2_gap['last_day_hour'] = every_hour_time2_gap.groupby('UID')['hour_'].shift(1)\n",
    "every_hour_time2_gap['every_hour_time2_gap'] = every_hour_time2_gap['hour_'] - every_hour_time2_gap['last_day_hour']\n",
    "every_hour_time2_gap = every_hour_time2_gap[['UID', 'every_hour_time2_gap']]\n",
    "every_hour_time2_gap = every_hour_time2_gap.groupby('UID')['every_hour_time2_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_hour_time2_gap.columns = ['UID', 'every_hour_time2_gap_sum', 'every_hour_time2_gap_std', 'every_hour_time2_gap_mean', 'every_hour_time2_gap_min', 'every_hour_time2_gap_max']\n",
    "data = pd.merge(data, every_hour_time2_gap, on='UID', how='left')\n",
    "\n",
    "# 每一天每一分钟用户操作的时间差\n",
    "every_minute_time2_gap = transaction_data[['UID', 'day', 'time']]\n",
    "every_minute_time2_gap['minute'] = every_minute_time2_gap[['day', 'time']].apply(lambda x: 1440 * (x['day'] - 1) + 60 * int(x['time'].split(':')[0]) + int(x['time'].split(':')[1]), axis=1)\n",
    "every_minute_time2_gap = every_minute_time2_gap[['UID', 'minute']]\n",
    "every_minute_time2_gap = every_minute_time2_gap.sort_values(['UID', 'minute'])\n",
    "every_minute_time2_gap['last_minute'] = every_minute_time2_gap.groupby('UID')['minute'].shift(1)\n",
    "every_minute_time2_gap['last_minute_gap'] = every_minute_time2_gap['minute'] - every_minute_time2_gap['last_minute']\n",
    "every_minute_time2_gap = every_minute_time2_gap[['UID', 'last_minute_gap']]\n",
    "every_minute_time2_gap = every_minute_time2_gap.groupby('UID')['last_minute_gap'].agg({'sum', 'mean', 'std', 'max', 'min'}).reset_index()\n",
    "data = pd.merge(data, every_minute_time2_gap, on='UID', how='left')\n",
    "\n",
    "# 每三天用户操作的时间差\n",
    "every_three_day_time2_gap = transaction_data[['UID', 'day']]\n",
    "every_three_day_time2_gap = every_three_day_time2_gap.sort_values(['UID', 'day'])\n",
    "every_three_day_time2_gap['last_three_day'] = every_three_day_time2_gap.groupby('UID')['day'].shift(3)\n",
    "every_three_day_time2_gap['last_three_day_gap'] = every_three_day_time2_gap['day'] - every_three_day_time2_gap['last_three_day']\n",
    "every_three_day_time2_gap = every_three_day_time2_gap[['UID', 'last_three_day_gap']]\n",
    "every_three_day_time2_gap = every_three_day_time2_gap.groupby('UID')['last_three_day_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_three_day_time2_gap.columns = ['UID', 'every_three_day_time2_gap_sum', 'every_three_day_time2_gap_std', 'every_three_day_time2_gap_min', 'every_three_day_time2_gap_mean', 'every_three_day_time2_gap_max']\n",
    "data = pd.merge(data, every_three_day_time2_gap, on='UID', how='left')\n",
    "\n",
    "# 每七天用户操作的时间差\n",
    "every_sixteen_day_time2_gap = transaction_data[['UID', 'day']]\n",
    "every_sixteen_day_time2_gap = every_sixteen_day_time2_gap.sort_values(['UID', 'day'])\n",
    "every_sixteen_day_time2_gap['last_sixteen_day'] = every_sixteen_day_time2_gap.groupby('UID')['day'].shift(7)\n",
    "every_sixteen_day_time2_gap['last_sixteen_day_gap'] = every_sixteen_day_time2_gap['day'] - every_sixteen_day_time2_gap['last_sixteen_day']\n",
    "every_sixteen_day_time2_gap = every_sixteen_day_time2_gap[['UID', 'last_sixteen_day_gap']]\n",
    "every_sixteen_day_time2_gap = every_sixteen_day_time2_gap.groupby('UID')['last_sixteen_day_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "every_sixteen_day_time2_gap.columns = ['UID', 'every_sixteen_day_time2_gap_sum', 'every_sixteen_day_time2_gap_std', 'every_sixteen_day_time2_gap_min', 'every_sixteen_day_time2_gap_mean', 'every_sixteen_day_time2_gap_max']\n",
    "data = pd.merge(data, every_sixteen_day_time2_gap, on='UID', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mac1 is over!\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-36-48616bfdbd6d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m    228\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    229\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[1;32min\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;34m'mac1'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'ip1_sub'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'trans_type1'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'market_code'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 230\u001b[1;33m     \u001b[0mdata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_UID_time_label\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtransaction_data\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-36-48616bfdbd6d>\u001b[0m in \u001b[0;36mget_UID_time_label\u001b[1;34m(data, x, transaction_data)\u001b[0m\n\u001b[0;32m    120\u001b[0m     \u001b[0mmerchant_every_work_time_rate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmerchant_every_work_time_rate\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'UID'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mx\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m'_merchant_every_work_time_rate'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    121\u001b[0m     \u001b[0mmerchant_every_work_time_rate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmerchant_every_work_time_rate\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgroupby\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'UID'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mx\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m'_merchant_every_work_time_rate'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0magg\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'sum'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreset_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 122\u001b[1;33m     \u001b[0mdata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmerge\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmerchant_every_work_time_rate\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mon\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'UID'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhow\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'left'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    123\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    124\u001b[0m     \u001b[1;31m# 用户merchant每天下午的统计量\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\hasee\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\pandas\\core\\reshape\\merge.py\u001b[0m in \u001b[0;36mmerge\u001b[1;34m(left, right, how, on, left_on, right_on, left_index, right_index, sort, suffixes, copy, indicator, validate)\u001b[0m\n\u001b[0;32m     60\u001b[0m                          \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mindicator\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mindicator\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     61\u001b[0m                          validate=validate)\n\u001b[1;32m---> 62\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mop\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_result\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     63\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     64\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\hasee\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\pandas\\core\\reshape\\merge.py\u001b[0m in \u001b[0;36mget_result\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    580\u001b[0m             \u001b[1;33m[\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mldata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlindexers\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mrdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrindexers\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    581\u001b[0m             \u001b[0maxes\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mllabels\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrlabels\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mjoin_index\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 582\u001b[1;33m             concat_axis=0, copy=self.copy)\n\u001b[0m\u001b[0;32m    583\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    584\u001b[0m         \u001b[0mtyp\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mleft\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_constructor\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\hasee\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\pandas\\core\\internals.py\u001b[0m in \u001b[0;36mconcatenate_block_managers\u001b[1;34m(mgrs_indexers, axes, concat_axis, copy)\u001b[0m\n\u001b[0;32m   5419\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   5420\u001b[0m             b = make_block(\n\u001b[1;32m-> 5421\u001b[1;33m                 \u001b[0mconcatenate_join_units\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjoin_units\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconcat_axis\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   5422\u001b[0m                 placement=placement)\n\u001b[0;32m   5423\u001b[0m         \u001b[0mblocks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\hasee\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\pandas\\core\\internals.py\u001b[0m in \u001b[0;36mconcatenate_join_units\u001b[1;34m(join_units, concat_axis, copy)\u001b[0m\n\u001b[0;32m   5563\u001b[0m     to_concat = [ju.get_reindexed_values(empty_dtype=empty_dtype,\n\u001b[0;32m   5564\u001b[0m                                          upcasted_na=upcasted_na)\n\u001b[1;32m-> 5565\u001b[1;33m                  for ju in join_units]\n\u001b[0m\u001b[0;32m   5566\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   5567\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mto_concat\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\hasee\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\pandas\\core\\internals.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m   5563\u001b[0m     to_concat = [ju.get_reindexed_values(empty_dtype=empty_dtype,\n\u001b[0;32m   5564\u001b[0m                                          upcasted_na=upcasted_na)\n\u001b[1;32m-> 5565\u001b[1;33m                  for ju in join_units]\n\u001b[0m\u001b[0;32m   5566\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   5567\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mto_concat\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\hasee\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\pandas\\core\\internals.py\u001b[0m in \u001b[0;36mget_reindexed_values\u001b[1;34m(self, empty_dtype, upcasted_na)\u001b[0m\n\u001b[0;32m   5873\u001b[0m             \u001b[1;32mfor\u001b[0m \u001b[0max\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mindexer\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindexers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   5874\u001b[0m                 values = algos.take_nd(values, indexer, axis=ax,\n\u001b[1;32m-> 5875\u001b[1;33m                                        fill_value=fill_value)\n\u001b[0m\u001b[0;32m   5876\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   5877\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mvalues\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "def get_UID_time_label(data, x, transaction_data):\n",
    "    \"\"\"\n",
    "    UID与x和trans_amt交叉的时间频率\n",
    "    \"\"\"\n",
    "    # 用户merchant每天每小时的统计\n",
    "    merchant_every_hour = transaction_data[['UID', x, 'day', 'hour', 'trans_amt']]\n",
    "    merchant_every_hour = merchant_every_hour.groupby(['UID', x, 'day', 'hour'])['trans_amt'].agg('sum').reset_index()\n",
    "    merchant_every_hour = merchant_every_hour[['UID', 'trans_amt']]\n",
    "    merchant_every_hour = merchant_every_hour.groupby('UID')['trans_amt'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    merchant_every_hour.columns = ['UID', x + '_merchant_every_hour_std', x + '_merchant_every_hour_mean', x + '_merchant_every_hour_min', x + '_merchant_every_hour_max']\n",
    "    data = pd.merge(data, merchant_every_hour, on='UID', how='left')\n",
    "\n",
    "    # 用户merchant每天每小时的gap\n",
    "    merchant_every_hour_gap = transaction_data[['UID', x, 'day', 'hour', 'trans_amt']]\n",
    "    merchant_every_hour_gap = merchant_every_hour_gap.groupby(['UID', x, 'day', 'hour'])['trans_amt'].agg('sum').reset_index()\n",
    "    merchant_every_hour_gap = merchant_every_hour_gap[['UID', 'trans_amt']]\n",
    "    merchant_every_hour_gap['trans_amt_before'] = merchant_every_hour_gap.groupby('UID')['trans_amt'].shift(1)\n",
    "    merchant_every_hour_gap['trans_amt_hour_gap'] = merchant_every_hour_gap['trans_amt'] - merchant_every_hour_gap['trans_amt_before']\n",
    "    merchant_every_hour_gap = merchant_every_hour_gap[['UID', 'trans_amt_hour_gap']]\n",
    "    merchant_every_hour_gap = merchant_every_hour_gap.groupby('UID')['trans_amt_hour_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    merchant_every_hour_gap.columns = ['UID', x + '_merchant_every_hour_gap_sum', x + '_merchant_every_hour_gap_std', x + '_merchant_every_hour_gap_mean', x + '_merchant_every_hour_gap_min', x + '_merchant_every_hour_gap_max']\n",
    "    data = pd.merge(data, merchant_every_hour_gap, on='UID', how='left')\n",
    "\n",
    "    # 用户merchant每天每小时的比率和\n",
    "    merchant_every_hour_rate = transaction_data[['UID', x, 'day', 'hour', 'trans_amt']]\n",
    "    merchant_every_hour_rate = merchant_every_hour_rate.groupby(['UID', x, 'day', 'hour'])['trans_amt'].agg('sum').reset_index()\n",
    "    merchant_every_hour_rate = merchant_every_hour_rate[['UID', 'trans_amt']]\n",
    "    merchant_every_hour_rate['trans_amt_before'] = merchant_every_hour_rate.groupby('UID')['trans_amt'].shift(1)\n",
    "    merchant_every_hour_rate['trans_amt_hour_rate'] = merchant_every_hour_rate['trans_amt'] / merchant_every_hour_rate['trans_amt_before']\n",
    "    merchant_every_hour_rate = merchant_every_hour_rate[['UID', 'trans_amt_hour_rate']]\n",
    "    merchant_every_hour_rate = merchant_every_hour_rate.groupby('UID')['trans_amt_hour_rate'].agg('sum').reset_index()\n",
    "    data = pd.merge(data, merchant_every_hour_rate, on='UID', how='left')\n",
    "\n",
    "\n",
    "    # 用户merchant每天的统计\n",
    "    merchant_every_day = transaction_data[['UID', x, 'day', 'trans_amt']]\n",
    "    merchant_every_day = merchant_every_day.groupby(['UID', x, 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "    merchant_every_day = merchant_every_day[['UID', 'trans_amt']]\n",
    "    merchant_every_day = merchant_every_day.groupby('UID')['trans_amt'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    merchant_every_day.columns = ['UID', x + '_merchant_every_day_std', x + '_merchant_every_day_mean', x + '_merchant_every_day_min', x + '_merchant_every_day_max']\n",
    "    data = pd.merge(data, merchant_every_day, on='UID', how='left')\n",
    "\n",
    "    # 用户merchant每天的gap\n",
    "    merchant_every_day_gap = transaction_data[['UID', x, 'day', 'trans_amt']]\n",
    "    merchant_every_day_gap = merchant_every_day_gap.groupby(['UID', x, 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "    merchant_every_day_gap = merchant_every_day_gap[['UID', 'trans_amt']]\n",
    "    merchant_every_day_gap['trans_amt_before'] = merchant_every_day_gap.groupby('UID')['trans_amt'].shift(1)\n",
    "    merchant_every_day_gap['trans_amt_day_gap'] = merchant_every_day_gap['trans_amt'] - merchant_every_day_gap['trans_amt_before']\n",
    "    merchant_every_day_gap = merchant_every_day_gap[['UID', 'trans_amt_day_gap']]\n",
    "    merchant_every_day_gap = merchant_every_day_gap.groupby('UID')['trans_amt_day_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    merchant_every_day_gap.columns = ['UID', x + '_merchant_every_day_gap_sum', x + '_merchant_every_day_gap_mean', x + '_merchant_every_day_gap_max', x + '_merchant_every_day_gap_min', x + '_merchant_every_day_gap_std']\n",
    "    data = pd.merge(data, merchant_every_day_gap, on='UID', how='left')\n",
    "\n",
    "    # 用户merchant每天的rate\n",
    "    merchant_every_day_rate = transaction_data[['UID', x, 'day', 'trans_amt']]\n",
    "    merchant_every_day_rate = merchant_every_day_rate.groupby(['UID', x, 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "    merchant_every_day_rate = merchant_every_day_rate[['UID', 'trans_amt']]\n",
    "    merchant_every_day_rate['trans_amt_before'] = merchant_every_day_rate.groupby('UID')['trans_amt'].shift(1)\n",
    "    merchant_every_day_rate['trans_amt_day_rate'] = merchant_every_day_rate['trans_amt'] / merchant_every_day_rate['trans_amt_before']\n",
    "    merchant_every_day_rate = merchant_every_day_rate[['UID', 'trans_amt_day_rate']]\n",
    "    merchant_every_day_rate = merchant_every_day_rate.groupby('UID')['trans_amt_day_rate'].agg('sum').reset_index()\n",
    "    data = pd.merge(data, merchant_every_day_rate, on='UID', how='left')\n",
    "\n",
    "\n",
    "    # 用户merchant每天半夜的统计量\n",
    "    merchant_every_morning = transaction_data[transaction_data.hour <= 5][['UID', x, 'day', 'trans_amt']]\n",
    "    merchant_every_morning = merchant_every_morning.groupby(['UID', x, 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "    merchant_every_morning = merchant_every_morning[['UID', 'trans_amt']]\n",
    "    merchant_every_morning = merchant_every_morning.groupby('UID')['trans_amt'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    merchant_every_morning.columns = ['UID', x + '_merchant_every_morning_std', x + '_merchant_every_morning_mean', x + '_merchant_every_morning_min', x + '_merchant_every_morning_max']\n",
    "    data = pd.merge(data, merchant_every_morning, on='UID', how='left')\n",
    "\n",
    "    # 用户merchant每天半夜的gap\n",
    "    merchant_every_morning_gap = transaction_data[transaction_data.hour <= 5][['UID', x, 'day', 'trans_amt']]\n",
    "    merchant_every_morning_gap = merchant_every_morning_gap.groupby(['UID', x, 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "    merchant_every_morning_gap = merchant_every_morning_gap[['UID', 'trans_amt']]\n",
    "    merchant_every_morning_gap['trans_amt_before'] = merchant_every_morning_gap.groupby('UID')['trans_amt'].shift(1)\n",
    "    merchant_every_morning_gap[x + '_merchant_every_morning_gap'] = merchant_every_morning_gap['trans_amt'] - merchant_every_morning_gap['trans_amt_before']\n",
    "    merchant_every_morning_gap = merchant_every_morning_gap[['UID', x + '_merchant_every_morning_gap']]\n",
    "    merchant_every_morning_gap = merchant_every_morning_gap.groupby('UID')[x + '_merchant_every_morning_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    merchant_every_morning_gap.columns = ['UID', x + '_merchant_every_morning_gap_sum', x + '_merchant_every_morning_gap_std', x + '_merchant_every_morning_gap_mean', x + '_merchant_every_morning_gap_min', x + '_merchant_every_morning_gap_max']\n",
    "    data = pd.merge(data, merchant_every_morning_gap, on='UID', how='left')\n",
    "\n",
    "    # 用户merchant每天半夜的rate\n",
    "    merchant_every_morning_rate = transaction_data[transaction_data.hour <= 5][['UID', x, 'day', 'trans_amt']]\n",
    "    merchant_every_morning_rate = merchant_every_morning_rate.groupby(['UID', x, 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "    merchant_every_morning_rate = merchant_every_morning_rate[['UID', 'trans_amt']]\n",
    "    merchant_every_morning_rate['trans_amt_before'] = merchant_every_morning_rate.groupby('UID')['trans_amt'].shift(1)\n",
    "    merchant_every_morning_rate[x + '_merchant_every_morning_rate'] = merchant_every_morning_rate['trans_amt'] / merchant_every_morning_rate['trans_amt_before']\n",
    "    merchant_every_morning_rate = merchant_every_morning_rate[['UID', x + '_merchant_every_morning_rate']]\n",
    "    merchant_every_morning_rate = merchant_every_morning_rate.groupby('UID')[x + '_merchant_every_morning_rate'].agg('sum').reset_index()\n",
    "    data = pd.merge(data, merchant_every_morning_rate, on='UID', how='left')\n",
    "\n",
    "\n",
    "    # 用户merchant每天早上的统计量\n",
    "    merchant_every_work_time = transaction_data[(transaction_data.hour >= 6)&(transaction_data.hour <= 11)][['UID', x, 'day', 'trans_amt']]\n",
    "    merchant_every_work_time = merchant_every_work_time.groupby(['UID', x, 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "    merchant_every_work_time = merchant_every_work_time[['UID', 'trans_amt']]\n",
    "    merchant_every_work_time = merchant_every_work_time.groupby('UID')['trans_amt'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    merchant_every_work_time.columns = ['UID', x + '_merchant_every_work_time_std', x + '_merchant_every_work_time_mean', x + '_merchant_every_work_time_min', x + '_merchant_every_work_time_max']\n",
    "    data = pd.merge(data, merchant_every_work_time, on='UID', how='left')\n",
    "\n",
    "    # 用户merchant每天早上的gap\n",
    "    merchant_every_work_time_gap = transaction_data[(transaction_data.hour >= 6)&(transaction_data.hour <= 11)][['UID', x, 'day', 'trans_amt']]\n",
    "    merchant_every_work_time_gap = merchant_every_work_time_gap.groupby(['UID', x, 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "    merchant_every_work_time_gap = merchant_every_work_time_gap[['UID', 'trans_amt']]\n",
    "    merchant_every_work_time_gap['trans_amt_before'] = merchant_every_work_time_gap.groupby('UID')['trans_amt'].shift(1)\n",
    "    merchant_every_work_time_gap[x + '_merchant_every_work_time_gap'] = merchant_every_work_time_gap['trans_amt'] - merchant_every_work_time_gap['trans_amt_before']\n",
    "    merchant_every_work_time_gap = merchant_every_work_time_gap[['UID', x + '_merchant_every_work_time_gap']]\n",
    "    merchant_every_work_time_gap = merchant_every_work_time_gap.groupby('UID')[x + '_merchant_every_work_time_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    merchant_every_work_time_gap.columns = ['UID', x + '_merchant_every_work_time_gap_sum', x + '_merchant_every_work_time_gap_std', x + '_merchant_every_work_time_gap_mean', x + '_merchant_every_work_time_gap_min', x + '_merchant_every_work_time_gap_max']\n",
    "    data = pd.merge(data, merchant_every_work_time_gap, on='UID', how='left')\n",
    "\n",
    "    # 用户merchant每天早上的rate\n",
    "    merchant_every_work_time_rate = transaction_data[(transaction_data.hour >= 6)&(transaction_data.hour <= 11)][['UID', x, 'day', 'trans_amt']]\n",
    "    merchant_every_work_time_rate = merchant_every_work_time_rate.groupby(['UID', x, 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "    merchant_every_work_time_rate = merchant_every_work_time_rate[['UID', 'trans_amt']]\n",
    "    merchant_every_work_time_rate['trans_amt_before'] = merchant_every_work_time_rate.groupby('UID')['trans_amt'].shift(1)\n",
    "    merchant_every_work_time_rate[x + '_merchant_every_work_time_rate'] = merchant_every_work_time_rate['trans_amt'] / merchant_every_work_time_rate['trans_amt_before']\n",
    "    merchant_every_work_time_rate = merchant_every_work_time_rate[['UID', x + '_merchant_every_work_time_rate']]\n",
    "    merchant_every_work_time_rate = merchant_every_work_time_rate.groupby('UID')[x + '_merchant_every_work_time_rate'].agg('sum').reset_index()\n",
    "    data = pd.merge(data, merchant_every_work_time_rate, on='UID', how='left')\n",
    "\n",
    "    # 用户merchant每天下午的统计量\n",
    "    merchant_every_afternoon = transaction_data[(transaction_data.hour >= 12)&(transaction_data.hour <= 17)][['UID', x, 'day', 'trans_amt']]\n",
    "    merchant_every_afternoon = merchant_every_afternoon.groupby(['UID', x, 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "    merchant_every_afternoon = merchant_every_afternoon[['UID', 'trans_amt']]\n",
    "    merchant_every_afternoon = merchant_every_afternoon.groupby('UID')['trans_amt'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    merchant_every_afternoon.columns = ['UID', x + '_merchant_every_afternoon_std', x + '_merchant_every_afternoon_mean', x + '_merchant_every_afternoon_min', x + '_merchant_every_afternoon_max']\n",
    "    data = pd.merge(data, merchant_every_afternoon, on='UID', how='left')\n",
    "\n",
    "    # 用户merchant每天下午的gap\n",
    "    merchant_every_afternoon_gap = transaction_data[(transaction_data.hour >= 12)&(transaction_data.hour <= 17)][['UID', x, 'day', 'trans_amt']]\n",
    "    merchant_every_afternoon_gap = merchant_every_afternoon_gap.groupby(['UID', x, 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "    merchant_every_afternoon_gap = merchant_every_afternoon_gap[['UID', 'trans_amt']]\n",
    "    merchant_every_afternoon_gap['trans_amt_before'] = merchant_every_afternoon_gap.groupby('UID')['trans_amt'].shift(1)\n",
    "    merchant_every_afternoon_gap[x + '_merchant_every_afternoon_gap'] = merchant_every_afternoon_gap['trans_amt'] - merchant_every_afternoon_gap['trans_amt_before']\n",
    "    merchant_every_afternoon_gap = merchant_every_afternoon_gap[['UID', x + '_merchant_every_afternoon_gap']]\n",
    "    merchant_every_afternoon_gap = merchant_every_afternoon_gap.groupby('UID')[x + '_merchant_every_afternoon_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    merchant_every_afternoon_gap.columns = ['UID', x + '_merchant_every_afternoon_gap_sum', x + '_merchant_every_afternoon_gap_std', x + '_merchant_every_afternoon_gap_mean', x + '_merchant_every_afternoon_gap_min', x + '_merchant_every_afternoon_gap_max']\n",
    "    data = pd.merge(data, merchant_every_afternoon_gap, on='UID', how='left')\n",
    "\n",
    "    # 用户merchant每天下午的rate\n",
    "    merchant_every_afternoon_rate = transaction_data[(transaction_data.hour >= 12)&(transaction_data.hour <= 17)][['UID', x, 'day', 'trans_amt']]\n",
    "    merchant_every_afternoon_rate = merchant_every_afternoon_rate.groupby(['UID', x, 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "    merchant_every_afternoon_rate = merchant_every_afternoon_rate[['UID', 'trans_amt']]\n",
    "    merchant_every_afternoon_rate['trans_amt_before'] = merchant_every_afternoon_rate.groupby('UID')['trans_amt'].shift(1)\n",
    "    merchant_every_afternoon_rate[x + '_merchant_every_afternoon_rate'] = merchant_every_afternoon_rate['trans_amt'] / merchant_every_afternoon_rate['trans_amt_before']\n",
    "    merchant_every_afternoon_rate = merchant_every_afternoon_rate[['UID', x + '_merchant_every_afternoon_rate']]\n",
    "    merchant_every_afternoon_rate = merchant_every_afternoon_rate.groupby('UID')[x + '_merchant_every_afternoon_rate'].agg('sum').reset_index()\n",
    "    data = pd.merge(data, merchant_every_afternoon_rate, on='UID', how='left')\n",
    "\n",
    "    # 用户merchant每天晚上的统计量\n",
    "    merchant_every_night = transaction_data[(transaction_data.hour >= 18)&(transaction_data.hour <= 23)][['UID', x, 'day', 'trans_amt']]\n",
    "    merchant_every_night = merchant_every_night.groupby(['UID', x, 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "    merchant_every_night = merchant_every_night[['UID', 'trans_amt']]\n",
    "    merchant_every_night = merchant_every_night.groupby('UID')['trans_amt'].agg({'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    merchant_every_night.columns = ['UID', x + '_merchant_every_night_std', x + '_merchant_every_night_mean', x + '_merchant_every_night_min', x + '_merchant_every_night_max']\n",
    "    data = pd.merge(data, merchant_every_night, on='UID', how='left')\n",
    "\n",
    "    # 用户merchant每天晚上的gap\n",
    "    merchant_every_night_gap = transaction_data[(transaction_data.hour >= 18)&(transaction_data.hour <= 23)][['UID', x, 'day', 'trans_amt']]\n",
    "    merchant_every_night_gap = merchant_every_night_gap.groupby(['UID', x, 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "    merchant_every_night_gap = merchant_every_night_gap[['UID', 'trans_amt']]\n",
    "    merchant_every_night_gap['trans_amt_before'] = merchant_every_night_gap.groupby('UID')['trans_amt'].shift(1)\n",
    "    merchant_every_night_gap[x + '_merchant_every_night_gap'] = merchant_every_night_gap['trans_amt'] - merchant_every_night_gap['trans_amt_before']\n",
    "    merchant_every_night_gap = merchant_every_night_gap[['UID', x + '_merchant_every_night_gap']]\n",
    "    merchant_every_night_gap = merchant_every_night_gap.groupby('UID')[x + '_merchant_every_night_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    merchant_every_night_gap.columns = ['UID', x + '_merchant_every_night_gap_sum', x + '_merchant_every_night_gap_std', x + '_merchant_every_night_gap_mean', x + '_merchant_every_night_gap_min', x + '_merchant_every_night_gap_max']\n",
    "    data = pd.merge(data, merchant_every_night_gap, on='UID', how='left')\n",
    "\n",
    "    # 用户merchant每天晚上的rate\n",
    "    merchant_every_night_rate = transaction_data[(transaction_data.hour >= 18)&(transaction_data.hour <= 23)][['UID', x, 'day', 'trans_amt']]\n",
    "    merchant_every_night_rate = merchant_every_night_rate.groupby(['UID', x, 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "    merchant_every_night_rate = merchant_every_night_rate[['UID', 'trans_amt']]\n",
    "    merchant_every_night_rate['trans_amt_before'] = merchant_every_night_rate.groupby('UID')['trans_amt'].shift(1)\n",
    "    merchant_every_night_rate[x + '_merchant_every_night_rate'] = merchant_every_night_rate['trans_amt'] / merchant_every_night_rate['trans_amt_before']\n",
    "    merchant_every_night_rate = merchant_every_night_rate[['UID', x + '_merchant_every_night_rate']]\n",
    "    merchant_every_night_rate = merchant_every_night_rate.groupby('UID')[x + '_merchant_every_night_rate'].agg('sum').reset_index()\n",
    "    data = pd.merge(data, merchant_every_night_rate, on='UID', how='left')\n",
    "\n",
    "\n",
    "    # 用户merchant每三天的gap\n",
    "    merchant_every_three_day_gap = transaction_data[['UID', x, 'day', 'trans_amt']]\n",
    "    merchant_every_three_day_gap = merchant_every_three_day_gap.groupby(['UID', x, 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "    merchant_every_three_day_gap = merchant_every_three_day_gap[['UID', 'trans_amt']]\n",
    "    merchant_every_three_day_gap['trans_amt_before'] = merchant_every_three_day_gap.groupby('UID')['trans_amt'].shift(3)\n",
    "    merchant_every_three_day_gap['trans_amt_three_day_gap'] = merchant_every_three_day_gap['trans_amt'] - merchant_every_three_day_gap['trans_amt_before']\n",
    "    merchant_every_three_day_gap = merchant_every_three_day_gap[['UID', 'trans_amt_three_day_gap']]\n",
    "    merchant_every_three_day_gap = merchant_every_three_day_gap.groupby('UID')['trans_amt_three_day_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    merchant_every_three_day_gap.columns = ['UID', x + '_merchant_every_three_day_gap_sum', x + '_merchant_every_three_day_gap_std', x + '_merchant_every_three_day_gap_mean', x + '_merchant_every_three_day_gap_min', x + '_merchant_every_three_day_gap_max']\n",
    "    data = pd.merge(data, merchant_every_three_day_gap, on='UID', how='left')\n",
    "\n",
    "    # 用户merchant每三天的rate\n",
    "    merchant_every_three_day_rate = transaction_data[['UID', x, 'day', 'trans_amt']]\n",
    "    merchant_every_three_day_rate = merchant_every_three_day_rate.groupby(['UID', x, 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "    merchant_every_three_day_rate = merchant_every_three_day_rate[['UID', 'trans_amt']]\n",
    "    merchant_every_three_day_rate['trans_amt_before'] = merchant_every_three_day_rate.groupby('UID')['trans_amt'].shift(3)\n",
    "    merchant_every_three_day_rate['trans_amt_three_day_rate'] = merchant_every_three_day_rate['trans_amt'] / merchant_every_three_day_rate['trans_amt_before']\n",
    "    merchant_every_three_day_rate = merchant_every_three_day_rate[['UID', 'trans_amt_three_day_rate']]\n",
    "    merchant_every_three_day_rate = merchant_every_three_day_rate.groupby('UID')['trans_amt_three_day_rate'].agg('sum').reset_index()\n",
    "    data = pd.merge(data, merchant_every_three_day_rate, on='UID', how='left')\n",
    "\n",
    "\n",
    "    # 用户merchant每七天的gap\n",
    "    merchant_every_sixteen_day_gap = transaction_data[['UID', x, 'day', 'trans_amt']]\n",
    "    merchant_every_sixteen_day_gap = merchant_every_sixteen_day_gap.groupby(['UID', x, 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "    merchant_every_sixteen_day_gap = merchant_every_sixteen_day_gap[['UID', 'trans_amt']]\n",
    "    merchant_every_sixteen_day_gap['trans_amt_before'] = merchant_every_sixteen_day_gap.groupby('UID')['trans_amt'].shift(7)\n",
    "    merchant_every_sixteen_day_gap['trans_amt_sixteen_day_gap'] = merchant_every_sixteen_day_gap['trans_amt'] - merchant_every_sixteen_day_gap['trans_amt_before']\n",
    "    merchant_every_sixteen_day_gap = merchant_every_sixteen_day_gap[['UID', 'trans_amt_sixteen_day_gap']]\n",
    "    merchant_every_sixteen_day_gap = merchant_every_sixteen_day_gap.groupby('UID')['trans_amt_sixteen_day_gap'].agg({'sum', 'mean', 'max', 'min', 'std'}).reset_index()\n",
    "    merchant_every_sixteen_day_gap.columns = ['UID', x + '_merchant_every_sixteen_day_gap_sum', x + '_merchant_every_sixteen_day_gap_std', x + '_merchant_every_sixteen_day_gap_mean', x + '_merchant_every_sixteen_day_gap_min', x + '_merchant_every_sixteen_day_gap_max']\n",
    "    data = pd.merge(data, merchant_every_sixteen_day_gap, on='UID', how='left')\n",
    "\n",
    "    # 用户merchant每七天的rate\n",
    "    merchant_every_sixteen_day_rate = transaction_data[['UID', x, 'day', 'trans_amt']]\n",
    "    merchant_every_sixteen_day_rate = merchant_every_sixteen_day_rate.groupby(['UID', x, 'day'])['trans_amt'].agg('sum').reset_index()\n",
    "    merchant_every_sixteen_day_rate = merchant_every_sixteen_day_rate[['UID', 'trans_amt']]\n",
    "    merchant_every_sixteen_day_rate['trans_amt_before'] = merchant_every_sixteen_day_rate.groupby('UID')['trans_amt'].shift(7)\n",
    "    merchant_every_sixteen_day_rate['trans_amt_sixteen_day_rate'] = merchant_every_sixteen_day_rate['trans_amt'] / merchant_every_sixteen_day_rate['trans_amt_before']\n",
    "    merchant_every_sixteen_day_rate = merchant_every_sixteen_day_rate[['UID', 'trans_amt_sixteen_day_rate']]\n",
    "    merchant_every_sixteen_day_rate = merchant_every_sixteen_day_rate.groupby('UID')['trans_amt_sixteen_day_rate'].agg('sum').reset_index()\n",
    "    data = pd.merge(data, merchant_every_sixteen_day_rate, on='UID', how='left')\n",
    "    \n",
    "    print(x + ' is over!')\n",
    "    return data\n",
    "\n",
    "for x in ['mac1', 'ip1_sub', 'trans_type1', 'market_code']:\n",
    "    data = get_UID_time_label(data, x, transaction_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "temp = data.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_two_count(data, t_data, x1, x2, t):\n",
    "    \"\"\"\n",
    "    获得x1和x2的交叉count\n",
    "    \"\"\"\n",
    "    temp = t_data[[x1, x2, 'UID']]\n",
    "    \n",
    "    operation_or_transaction_data = temp.copy()\n",
    "    operation_or_transaction_data.drop_duplicates(inplace=True)\n",
    "    \n",
    "    temp[x1 + '_' + x2 + '_count'] = temp['UID']\n",
    "    temp = temp.groupby([x1, x2])[x1 + '_' + x2 + '_count'].agg('nunique').reset_index()\n",
    "    \n",
    "    operation_or_transaction_data = pd.merge(operation_or_transaction_data, temp, on=[x1, x2], how='left')\n",
    "    operation_or_transaction_data[x1 + '_' + x2 + '_count_uid_' + t] = 1\n",
    "    operation_or_transaction_data = operation_or_transaction_data.groupby('UID')[x1 + '_' + x2 + '_count_uid_' + t].agg('sum').reset_index()\n",
    "    \n",
    "    data = pd.merge(data, operation_or_transaction_data, on='UID', how='left')\n",
    "    return data\n",
    "\n",
    "data = get_two_count(data, operation_data, 'ip1', 'geo_code', 'operation_data')\n",
    "data = get_two_count(data, operation_data, 'ip1', 'ip1_sub', 'operation_data')\n",
    "data = get_two_count(data, operation_data, 'ip1', 'wifi', 'operation_data')\n",
    "data = get_two_count(data, operation_data, 'geo_code', 'ip1_sub', 'operation_data')\n",
    "data = get_two_count(data, operation_data, 'geo_code', 'wifi', 'operation_data')\n",
    "data = get_two_count(data, operation_data, 'ip1_sub', 'wifi', 'operation_data')\n",
    "\n",
    "data = get_two_count(data, transaction_data, 'merchant', 'geo_code', 'transaction_data')\n",
    "data = get_two_count(data, transaction_data, 'merchant', 'ip1_sub', 'transaction_data')\n",
    "data = get_two_count(data, transaction_data, 'merchant', 'code1', 'transaction_data')\n",
    "data = get_two_count(data, transaction_data, 'geo_code', 'ip1_sub', 'transaction_data')\n",
    "data = get_two_count(data, transaction_data, 'geo_code', 'code1', 'transaction_data')\n",
    "data = get_two_count(data, transaction_data, 'ip1_sub', 'code1', 'transaction_data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_two_shuxing_count(data, t_data, x1, x2, t):\n",
    "    \"\"\"\n",
    "    groupby(x1)[x2].agg('nunique').reset_index()\n",
    "    \"\"\"\n",
    "    UID = t_data[['UID', x1]].drop_duplicates()\n",
    "    temp = t_data[[x1, x2]]\n",
    "    temp[x2 + '_of_nunique_' + x1] = temp[x2]\n",
    "    temp = temp.groupby(x1)[x2 + '_of_nunique_' + x1].agg('nunique').reset_index()\n",
    "    \n",
    "    UID = pd.merge(UID, temp, on=x1, how='left')\n",
    "    UID = UID.drop(x1, axis=1)\n",
    "    UID = UID.groupby('UID')[x2 + '_of_nunique_' + x1].agg('sum').reset_index()\n",
    "    \n",
    "    data = pd.merge(data, UID, on='UID', how='left')\n",
    "    return data\n",
    "    \n",
    "for x1 in ['ip1', 'geo_code', 'ip1_sub', 'wifi']:\n",
    "    for x2 in ['ip1', 'geo_code', 'ip1_sub', 'wifi']:\n",
    "        if x1 != x2:\n",
    "            data = get_two_shuxing_count(data, operation_data, x1, x2, 'operation_data')\n",
    "for x1 in ['device_code1', 'device_code2', 'device_code3']:\n",
    "    for x2 in ['ip1', 'geo_code', 'ip1_sub', 'wifi']:\n",
    "        data = get_two_shuxing_count(data, operation_data, x1, x2, 'operation_data')\n",
    "print('operation_data is over!')\n",
    "for x1 in ['merchant', 'geo_code', 'ip1_sub', 'code1']:\n",
    "    for x2 in ['merchant', 'geo_code', 'ip1_sub', 'code1']:\n",
    "        if x1 != x2:\n",
    "            data = get_two_shuxing_count(data, transaction_data, x1, x2, 'transaction_data')\n",
    "print('transaction_data is over!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_three_count(data, t_data, x1, x2, x3, t):\n",
    "    temp = t_data[[x1, x2, x3, 'UID']]\n",
    "    \n",
    "    operation_or_transaction_data = temp.copy()\n",
    "    operation_or_transaction_data.drop_duplicates(inplace=True)\n",
    "    \n",
    "    temp[x1 + '_' + x2 + '_' + x3 + '_count'] = temp['UID']\n",
    "    temp = temp.groupby([x1, x2, x3])[x1 + '_' + x2 + '_' + x3 + '_count'].agg('nunique').reset_index()\n",
    "    \n",
    "    operation_or_transaction_data = pd.merge(operation_or_transaction_data, temp, on=[x1, x2, x3], how='left')\n",
    "    operation_or_transaction_data[x1 + '_' + x2 + '_' + x3 + '_count'] = 1\n",
    "    operation_or_transaction_data = operation_or_transaction_data.groupby('UID')[x1 + '_' + x2 + '_' + x3 + '_count'].agg('sum').reset_index()\n",
    "    \n",
    "    print(operation_or_transaction_data)\n",
    "    data = pd.merge(data, operation_or_transaction_data, on='UID', how='left')\n",
    "    return data\n",
    "\n",
    "data = get_three_count(data, transaction_data, 'merchant', 'day', 'device1', 'transaction')\n",
    "data = get_three_count(data, transaction_data, 'merchant', 'day', 'device2', 'transaction')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Tag</th>\n",
       "      <th>UID</th>\n",
       "      <th>has_operation</th>\n",
       "      <th>has_transaction</th>\n",
       "      <th>operation_transaction_count_gap</th>\n",
       "      <th>success_rate</th>\n",
       "      <th>mode_success_max</th>\n",
       "      <th>mode_success_std</th>\n",
       "      <th>mode_success_sum</th>\n",
       "      <th>mode_success_min</th>\n",
       "      <th>...</th>\n",
       "      <th>mac1_merchant_every_three_day_gap_mean</th>\n",
       "      <th>mac1_merchant_every_three_day_gap_min</th>\n",
       "      <th>mac1_merchant_every_three_day_gap_max</th>\n",
       "      <th>trans_amt_three_day_rate_y</th>\n",
       "      <th>mac1_merchant_every_sixteen_day_gap_sum</th>\n",
       "      <th>mac1_merchant_every_sixteen_day_gap_std</th>\n",
       "      <th>mac1_merchant_every_sixteen_day_gap_mean</th>\n",
       "      <th>mac1_merchant_every_sixteen_day_gap_min</th>\n",
       "      <th>mac1_merchant_every_sixteen_day_gap_max</th>\n",
       "      <th>trans_amt_sixteen_day_rate_y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>10000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-7.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>10002</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-8.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>10003</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-11.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>10004</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-27.0</td>\n",
       "      <td>0.941176</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>8.333333</td>\n",
       "      <td>0.222222</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>10005</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1663 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Tag    UID  has_operation  has_transaction  \\\n",
       "0  1.0  10000              1                1   \n",
       "1  0.0  10002              1                1   \n",
       "2  0.0  10003              1                1   \n",
       "3  0.0  10004              1                1   \n",
       "4  0.0  10005              0                1   \n",
       "\n",
       "   operation_transaction_count_gap  success_rate  mode_success_max  \\\n",
       "0                             -7.0      1.000000               1.0   \n",
       "1                             -8.0      1.000000               1.0   \n",
       "2                            -11.0      1.000000               1.0   \n",
       "3                            -27.0      0.941176               1.0   \n",
       "4                             -1.0     -1.000000              -1.0   \n",
       "\n",
       "   mode_success_std  mode_success_sum  mode_success_min  \\\n",
       "0          1.000000          3.000000          0.000000   \n",
       "1          1.000000          7.000000          0.000000   \n",
       "2          1.000000          2.000000          0.000000   \n",
       "3          0.333333          8.333333          0.222222   \n",
       "4         -1.000000         -1.000000         -1.000000   \n",
       "\n",
       "               ...               mac1_merchant_every_three_day_gap_mean  \\\n",
       "0              ...                                                 -1.0   \n",
       "1              ...                                                  0.0   \n",
       "2              ...                                                  0.0   \n",
       "3              ...                                                  0.0   \n",
       "4              ...                                                  0.0   \n",
       "\n",
       "   mac1_merchant_every_three_day_gap_min  \\\n",
       "0                                   -1.0   \n",
       "1                                   -1.0   \n",
       "2                                   -1.0   \n",
       "3                                   -1.0   \n",
       "4                                   -1.0   \n",
       "\n",
       "   mac1_merchant_every_three_day_gap_max  trans_amt_three_day_rate_y  \\\n",
       "0                                   -1.0                        -1.0   \n",
       "1                                   -1.0                         0.0   \n",
       "2                                   -1.0                         0.0   \n",
       "3                                   -1.0                         0.0   \n",
       "4                                   -1.0                         0.0   \n",
       "\n",
       "   mac1_merchant_every_sixteen_day_gap_sum  \\\n",
       "0                                     -1.0   \n",
       "1                                     -1.0   \n",
       "2                                     -1.0   \n",
       "3                                     -1.0   \n",
       "4                                     -1.0   \n",
       "\n",
       "   mac1_merchant_every_sixteen_day_gap_std  \\\n",
       "0                                     -1.0   \n",
       "1                                     -1.0   \n",
       "2                                     -1.0   \n",
       "3                                     -1.0   \n",
       "4                                     -1.0   \n",
       "\n",
       "   mac1_merchant_every_sixteen_day_gap_mean  \\\n",
       "0                                      -1.0   \n",
       "1                                       0.0   \n",
       "2                                       0.0   \n",
       "3                                       0.0   \n",
       "4                                       0.0   \n",
       "\n",
       "   mac1_merchant_every_sixteen_day_gap_min  \\\n",
       "0                                     -1.0   \n",
       "1                                     -1.0   \n",
       "2                                     -1.0   \n",
       "3                                     -1.0   \n",
       "4                                     -1.0   \n",
       "\n",
       "   mac1_merchant_every_sixteen_day_gap_max  trans_amt_sixteen_day_rate_y  \n",
       "0                                     -1.0                          -1.0  \n",
       "1                                     -1.0                           0.0  \n",
       "2                                     -1.0                           0.0  \n",
       "3                                     -1.0                           0.0  \n",
       "4                                     -1.0                           0.0  \n",
       "\n",
       "[5 rows x 1663 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = data.fillna(-1)\n",
    "data.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train: (57389, 1661)\n",
      "train.columns: Index(['has_operation', 'has_transaction', 'operation_transaction_count_gap',\n",
      "       'success_rate', 'mode_success_max', 'mode_success_std',\n",
      "       'mode_success_sum', 'mode_success_min', 'mode_success_mean',\n",
      "       'day_nunique_of_operation',\n",
      "       ...\n",
      "       'mac1_merchant_every_three_day_gap_mean',\n",
      "       'mac1_merchant_every_three_day_gap_min',\n",
      "       'mac1_merchant_every_three_day_gap_max', 'trans_amt_three_day_rate_y',\n",
      "       'mac1_merchant_every_sixteen_day_gap_sum',\n",
      "       'mac1_merchant_every_sixteen_day_gap_std',\n",
      "       'mac1_merchant_every_sixteen_day_gap_mean',\n",
      "       'mac1_merchant_every_sixteen_day_gap_min',\n",
      "       'mac1_merchant_every_sixteen_day_gap_max',\n",
      "       'trans_amt_sixteen_day_rate_y'],\n",
      "      dtype='object', length=1661)\n",
      "test: (31588, 1661)\n"
     ]
    }
   ],
   "source": [
    "drop = ['UID', 'Tag']\n",
    "\n",
    "train = data[data.Tag != -1]\n",
    "test = data[data.Tag == -1]\n",
    "\n",
    "y_train = train.loc[:,'Tag']\n",
    "res = test.loc[:, ['UID']]\n",
    "\n",
    "train.drop(drop, axis=1, inplace=True)\n",
    "print('train:',train.shape)\n",
    "print('train.columns:',train.columns)\n",
    "\n",
    "test.drop(drop, axis=1, inplace=True)\n",
    "print('test:',test.shape)\n",
    "\n",
    "all_feature = [x for x in train.columns]# 所有的特征\n",
    "\n",
    "X_loc_train = train.values\n",
    "y_loc_train = y_train.values\n",
    "X_loc_test = test.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold 0\n",
      "[1]\ttrain's auc: 0.952989\tvalid's auc: 0.94214\n",
      "Training until validation scores don't improve for 100 rounds.\n",
      "[2]\ttrain's auc: 0.958286\tvalid's auc: 0.948767\n",
      "[3]\ttrain's auc: 0.961965\tvalid's auc: 0.951978\n",
      "[4]\ttrain's auc: 0.964586\tvalid's auc: 0.956656\n",
      "[5]\ttrain's auc: 0.964669\tvalid's auc: 0.956738\n",
      "[6]\ttrain's auc: 0.965246\tvalid's auc: 0.957904\n",
      "[7]\ttrain's auc: 0.966045\tvalid's auc: 0.958898\n",
      "[8]\ttrain's auc: 0.967934\tvalid's auc: 0.961831\n",
      "[9]\ttrain's auc: 0.968299\tvalid's auc: 0.962125\n",
      "[10]\ttrain's auc: 0.968438\tvalid's auc: 0.9627\n",
      "[11]\ttrain's auc: 0.968973\tvalid's auc: 0.963046\n",
      "[12]\ttrain's auc: 0.969297\tvalid's auc: 0.96309\n",
      "[13]\ttrain's auc: 0.972041\tvalid's auc: 0.966445\n",
      "[14]\ttrain's auc: 0.972717\tvalid's auc: 0.967588\n",
      "[15]\ttrain's auc: 0.972602\tvalid's auc: 0.967342\n",
      "[16]\ttrain's auc: 0.972945\tvalid's auc: 0.96782\n",
      "[17]\ttrain's auc: 0.972709\tvalid's auc: 0.967515\n",
      "[18]\ttrain's auc: 0.972856\tvalid's auc: 0.967663\n",
      "[19]\ttrain's auc: 0.973759\tvalid's auc: 0.968432\n",
      "[20]\ttrain's auc: 0.97425\tvalid's auc: 0.969194\n",
      "[21]\ttrain's auc: 0.975193\tvalid's auc: 0.969626\n",
      "[22]\ttrain's auc: 0.975269\tvalid's auc: 0.969658\n",
      "[23]\ttrain's auc: 0.975105\tvalid's auc: 0.969397\n",
      "[24]\ttrain's auc: 0.975325\tvalid's auc: 0.969105\n",
      "[25]\ttrain's auc: 0.975061\tvalid's auc: 0.968754\n",
      "[26]\ttrain's auc: 0.975474\tvalid's auc: 0.969207\n",
      "[27]\ttrain's auc: 0.975754\tvalid's auc: 0.969373\n",
      "[28]\ttrain's auc: 0.976101\tvalid's auc: 0.969551\n",
      "[29]\ttrain's auc: 0.976674\tvalid's auc: 0.969995\n",
      "[30]\ttrain's auc: 0.977545\tvalid's auc: 0.971061\n",
      "[31]\ttrain's auc: 0.977898\tvalid's auc: 0.971365\n",
      "[32]\ttrain's auc: 0.978143\tvalid's auc: 0.97172\n",
      "[33]\ttrain's auc: 0.978318\tvalid's auc: 0.971717\n",
      "[34]\ttrain's auc: 0.979038\tvalid's auc: 0.972878\n",
      "[35]\ttrain's auc: 0.979373\tvalid's auc: 0.973234\n",
      "[36]\ttrain's auc: 0.97945\tvalid's auc: 0.973282\n",
      "[37]\ttrain's auc: 0.979557\tvalid's auc: 0.973343\n",
      "[38]\ttrain's auc: 0.979805\tvalid's auc: 0.973651\n",
      "[39]\ttrain's auc: 0.980043\tvalid's auc: 0.973878\n",
      "[40]\ttrain's auc: 0.980222\tvalid's auc: 0.974168\n",
      "[41]\ttrain's auc: 0.980311\tvalid's auc: 0.97422\n",
      "[42]\ttrain's auc: 0.980457\tvalid's auc: 0.974371\n",
      "[43]\ttrain's auc: 0.980589\tvalid's auc: 0.974546\n",
      "[44]\ttrain's auc: 0.980877\tvalid's auc: 0.974707\n",
      "[45]\ttrain's auc: 0.981047\tvalid's auc: 0.975122\n",
      "[46]\ttrain's auc: 0.981202\tvalid's auc: 0.975241\n",
      "[47]\ttrain's auc: 0.981319\tvalid's auc: 0.975386\n",
      "[48]\ttrain's auc: 0.981424\tvalid's auc: 0.975505\n",
      "[49]\ttrain's auc: 0.981544\tvalid's auc: 0.975733\n",
      "[50]\ttrain's auc: 0.981675\tvalid's auc: 0.975756\n",
      "[51]\ttrain's auc: 0.981784\tvalid's auc: 0.97591\n",
      "[52]\ttrain's auc: 0.981922\tvalid's auc: 0.976059\n",
      "[53]\ttrain's auc: 0.982005\tvalid's auc: 0.976108\n",
      "[54]\ttrain's auc: 0.982172\tvalid's auc: 0.976389\n",
      "[55]\ttrain's auc: 0.982286\tvalid's auc: 0.976501\n",
      "[56]\ttrain's auc: 0.982495\tvalid's auc: 0.976658\n",
      "[57]\ttrain's auc: 0.982589\tvalid's auc: 0.976654\n",
      "[58]\ttrain's auc: 0.982761\tvalid's auc: 0.976855\n",
      "[59]\ttrain's auc: 0.982839\tvalid's auc: 0.977019\n",
      "[60]\ttrain's auc: 0.982928\tvalid's auc: 0.97709\n",
      "[61]\ttrain's auc: 0.983042\tvalid's auc: 0.97717\n",
      "[62]\ttrain's auc: 0.983199\tvalid's auc: 0.977325\n",
      "[63]\ttrain's auc: 0.983352\tvalid's auc: 0.977594\n",
      "[64]\ttrain's auc: 0.983462\tvalid's auc: 0.977701\n",
      "[65]\ttrain's auc: 0.983549\tvalid's auc: 0.977782\n",
      "[66]\ttrain's auc: 0.983719\tvalid's auc: 0.978012\n",
      "[67]\ttrain's auc: 0.983821\tvalid's auc: 0.978182\n",
      "[68]\ttrain's auc: 0.983962\tvalid's auc: 0.978357\n",
      "[69]\ttrain's auc: 0.984061\tvalid's auc: 0.97844\n",
      "[70]\ttrain's auc: 0.984152\tvalid's auc: 0.978564\n",
      "[71]\ttrain's auc: 0.984329\tvalid's auc: 0.97884\n",
      "[72]\ttrain's auc: 0.984478\tvalid's auc: 0.979029\n",
      "[73]\ttrain's auc: 0.98461\tvalid's auc: 0.979212\n",
      "[74]\ttrain's auc: 0.984644\tvalid's auc: 0.979234\n",
      "[75]\ttrain's auc: 0.984726\tvalid's auc: 0.979366\n",
      "[76]\ttrain's auc: 0.984817\tvalid's auc: 0.979471\n",
      "[77]\ttrain's auc: 0.984929\tvalid's auc: 0.979566\n",
      "[78]\ttrain's auc: 0.985029\tvalid's auc: 0.979647\n",
      "[79]\ttrain's auc: 0.985126\tvalid's auc: 0.979767\n",
      "[80]\ttrain's auc: 0.985236\tvalid's auc: 0.979924\n",
      "[81]\ttrain's auc: 0.985339\tvalid's auc: 0.980016\n",
      "[82]\ttrain's auc: 0.98543\tvalid's auc: 0.980149\n",
      "[83]\ttrain's auc: 0.985535\tvalid's auc: 0.98027\n",
      "[84]\ttrain's auc: 0.985643\tvalid's auc: 0.980347\n",
      "[85]\ttrain's auc: 0.98573\tvalid's auc: 0.980472\n",
      "[86]\ttrain's auc: 0.985786\tvalid's auc: 0.980568\n",
      "[87]\ttrain's auc: 0.985883\tvalid's auc: 0.980703\n",
      "[88]\ttrain's auc: 0.985976\tvalid's auc: 0.980757\n",
      "[89]\ttrain's auc: 0.986077\tvalid's auc: 0.980911\n",
      "[90]\ttrain's auc: 0.986137\tvalid's auc: 0.981017\n",
      "[91]\ttrain's auc: 0.98621\tvalid's auc: 0.981055\n",
      "[92]\ttrain's auc: 0.986326\tvalid's auc: 0.98118\n",
      "[93]\ttrain's auc: 0.98638\tvalid's auc: 0.98127\n",
      "[94]\ttrain's auc: 0.986436\tvalid's auc: 0.981341\n",
      "[95]\ttrain's auc: 0.986477\tvalid's auc: 0.981372\n",
      "[96]\ttrain's auc: 0.986564\tvalid's auc: 0.981461\n",
      "[97]\ttrain's auc: 0.986702\tvalid's auc: 0.981651\n",
      "[98]\ttrain's auc: 0.986794\tvalid's auc: 0.981692\n",
      "[99]\ttrain's auc: 0.986881\tvalid's auc: 0.981781\n",
      "[100]\ttrain's auc: 0.986975\tvalid's auc: 0.981843\n",
      "[101]\ttrain's auc: 0.987033\tvalid's auc: 0.981899\n",
      "[102]\ttrain's auc: 0.987115\tvalid's auc: 0.981957\n",
      "[103]\ttrain's auc: 0.987171\tvalid's auc: 0.982045\n",
      "[104]\ttrain's auc: 0.987235\tvalid's auc: 0.982066\n",
      "[105]\ttrain's auc: 0.987295\tvalid's auc: 0.982082\n",
      "[106]\ttrain's auc: 0.987366\tvalid's auc: 0.982172\n",
      "[107]\ttrain's auc: 0.987438\tvalid's auc: 0.982191\n",
      "[108]\ttrain's auc: 0.987485\tvalid's auc: 0.982295\n",
      "[109]\ttrain's auc: 0.987524\tvalid's auc: 0.982306\n",
      "[110]\ttrain's auc: 0.987564\tvalid's auc: 0.982325\n",
      "[111]\ttrain's auc: 0.987643\tvalid's auc: 0.982412\n",
      "[112]\ttrain's auc: 0.98769\tvalid's auc: 0.982493\n",
      "[113]\ttrain's auc: 0.987787\tvalid's auc: 0.982608\n",
      "[114]\ttrain's auc: 0.98784\tvalid's auc: 0.982688\n",
      "[115]\ttrain's auc: 0.987898\tvalid's auc: 0.982738\n",
      "[116]\ttrain's auc: 0.987945\tvalid's auc: 0.982807\n",
      "[117]\ttrain's auc: 0.987998\tvalid's auc: 0.982835\n",
      "[118]\ttrain's auc: 0.988059\tvalid's auc: 0.982869\n",
      "[119]\ttrain's auc: 0.988113\tvalid's auc: 0.982928\n",
      "[120]\ttrain's auc: 0.988146\tvalid's auc: 0.982915\n",
      "[121]\ttrain's auc: 0.988219\tvalid's auc: 0.982951\n",
      "[122]\ttrain's auc: 0.988325\tvalid's auc: 0.983014\n",
      "[123]\ttrain's auc: 0.988393\tvalid's auc: 0.983076\n",
      "[124]\ttrain's auc: 0.988434\tvalid's auc: 0.983066\n",
      "[125]\ttrain's auc: 0.98848\tvalid's auc: 0.983073\n",
      "[126]\ttrain's auc: 0.988535\tvalid's auc: 0.983088\n",
      "[127]\ttrain's auc: 0.988576\tvalid's auc: 0.983157\n",
      "[128]\ttrain's auc: 0.988625\tvalid's auc: 0.983175\n",
      "[129]\ttrain's auc: 0.988676\tvalid's auc: 0.983193\n",
      "[130]\ttrain's auc: 0.988731\tvalid's auc: 0.983236\n",
      "[131]\ttrain's auc: 0.988766\tvalid's auc: 0.983229\n",
      "[132]\ttrain's auc: 0.988813\tvalid's auc: 0.98325\n",
      "[133]\ttrain's auc: 0.988877\tvalid's auc: 0.983285\n",
      "[134]\ttrain's auc: 0.988927\tvalid's auc: 0.983317\n",
      "[135]\ttrain's auc: 0.989015\tvalid's auc: 0.983438\n",
      "[136]\ttrain's auc: 0.989065\tvalid's auc: 0.983482\n",
      "[137]\ttrain's auc: 0.989139\tvalid's auc: 0.983536\n",
      "[138]\ttrain's auc: 0.989241\tvalid's auc: 0.983587\n",
      "[139]\ttrain's auc: 0.989304\tvalid's auc: 0.983655\n",
      "[140]\ttrain's auc: 0.989374\tvalid's auc: 0.983708\n",
      "[141]\ttrain's auc: 0.989444\tvalid's auc: 0.983785\n",
      "[142]\ttrain's auc: 0.989496\tvalid's auc: 0.983814\n",
      "[143]\ttrain's auc: 0.989568\tvalid's auc: 0.983821\n",
      "[144]\ttrain's auc: 0.989614\tvalid's auc: 0.983868\n",
      "[145]\ttrain's auc: 0.989677\tvalid's auc: 0.983945\n",
      "[146]\ttrain's auc: 0.989725\tvalid's auc: 0.983999\n",
      "[147]\ttrain's auc: 0.989759\tvalid's auc: 0.98402\n",
      "[148]\ttrain's auc: 0.989815\tvalid's auc: 0.984065\n",
      "[149]\ttrain's auc: 0.989868\tvalid's auc: 0.984139\n",
      "[150]\ttrain's auc: 0.989925\tvalid's auc: 0.984152\n",
      "[151]\ttrain's auc: 0.989963\tvalid's auc: 0.984157\n",
      "[152]\ttrain's auc: 0.989996\tvalid's auc: 0.984167\n",
      "[153]\ttrain's auc: 0.990072\tvalid's auc: 0.98425\n",
      "[154]\ttrain's auc: 0.990105\tvalid's auc: 0.984243\n",
      "[155]\ttrain's auc: 0.990153\tvalid's auc: 0.984281\n",
      "[156]\ttrain's auc: 0.990197\tvalid's auc: 0.984284\n",
      "[157]\ttrain's auc: 0.990237\tvalid's auc: 0.984332\n",
      "[158]\ttrain's auc: 0.990276\tvalid's auc: 0.984347\n",
      "[159]\ttrain's auc: 0.990318\tvalid's auc: 0.984348\n",
      "[160]\ttrain's auc: 0.990365\tvalid's auc: 0.984363\n",
      "[161]\ttrain's auc: 0.990419\tvalid's auc: 0.98442\n",
      "[162]\ttrain's auc: 0.990457\tvalid's auc: 0.984443\n",
      "[163]\ttrain's auc: 0.990496\tvalid's auc: 0.984457\n",
      "[164]\ttrain's auc: 0.990537\tvalid's auc: 0.984479\n",
      "[165]\ttrain's auc: 0.990588\tvalid's auc: 0.984515\n",
      "[166]\ttrain's auc: 0.990612\tvalid's auc: 0.984502\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[167]\ttrain's auc: 0.990658\tvalid's auc: 0.984554\n",
      "[168]\ttrain's auc: 0.990684\tvalid's auc: 0.984564\n",
      "[169]\ttrain's auc: 0.990729\tvalid's auc: 0.984556\n",
      "[170]\ttrain's auc: 0.990762\tvalid's auc: 0.984537\n",
      "[171]\ttrain's auc: 0.990788\tvalid's auc: 0.984534\n",
      "[172]\ttrain's auc: 0.990834\tvalid's auc: 0.984554\n",
      "[173]\ttrain's auc: 0.990861\tvalid's auc: 0.984555\n",
      "[174]\ttrain's auc: 0.990887\tvalid's auc: 0.984551\n",
      "[175]\ttrain's auc: 0.990921\tvalid's auc: 0.984593\n",
      "[176]\ttrain's auc: 0.990959\tvalid's auc: 0.984612\n",
      "[177]\ttrain's auc: 0.990997\tvalid's auc: 0.98463\n",
      "[178]\ttrain's auc: 0.991036\tvalid's auc: 0.984639\n",
      "[179]\ttrain's auc: 0.991077\tvalid's auc: 0.984668\n",
      "[180]\ttrain's auc: 0.991109\tvalid's auc: 0.984677\n",
      "[181]\ttrain's auc: 0.991147\tvalid's auc: 0.984706\n",
      "[182]\ttrain's auc: 0.991188\tvalid's auc: 0.984697\n",
      "[183]\ttrain's auc: 0.991212\tvalid's auc: 0.9847\n",
      "[184]\ttrain's auc: 0.991246\tvalid's auc: 0.984718\n",
      "[185]\ttrain's auc: 0.991273\tvalid's auc: 0.984702\n",
      "[186]\ttrain's auc: 0.991297\tvalid's auc: 0.984797\n",
      "[187]\ttrain's auc: 0.991345\tvalid's auc: 0.984831\n",
      "[188]\ttrain's auc: 0.991382\tvalid's auc: 0.984869\n",
      "[189]\ttrain's auc: 0.991425\tvalid's auc: 0.984901\n",
      "[190]\ttrain's auc: 0.991467\tvalid's auc: 0.984914\n",
      "[191]\ttrain's auc: 0.991498\tvalid's auc: 0.984922\n",
      "[192]\ttrain's auc: 0.99154\tvalid's auc: 0.984955\n",
      "[193]\ttrain's auc: 0.99156\tvalid's auc: 0.98497\n",
      "[194]\ttrain's auc: 0.991601\tvalid's auc: 0.984998\n",
      "[195]\ttrain's auc: 0.991636\tvalid's auc: 0.984997\n",
      "[196]\ttrain's auc: 0.991678\tvalid's auc: 0.985009\n",
      "[197]\ttrain's auc: 0.991716\tvalid's auc: 0.985017\n",
      "[198]\ttrain's auc: 0.991752\tvalid's auc: 0.985055\n",
      "[199]\ttrain's auc: 0.991773\tvalid's auc: 0.985066\n",
      "[200]\ttrain's auc: 0.99181\tvalid's auc: 0.985096\n",
      "[201]\ttrain's auc: 0.99183\tvalid's auc: 0.985104\n",
      "[202]\ttrain's auc: 0.991852\tvalid's auc: 0.985131\n",
      "[203]\ttrain's auc: 0.991885\tvalid's auc: 0.985154\n",
      "[204]\ttrain's auc: 0.991927\tvalid's auc: 0.985181\n",
      "[205]\ttrain's auc: 0.991969\tvalid's auc: 0.985207\n",
      "[206]\ttrain's auc: 0.992014\tvalid's auc: 0.985231\n",
      "[207]\ttrain's auc: 0.992054\tvalid's auc: 0.985266\n",
      "[208]\ttrain's auc: 0.992082\tvalid's auc: 0.985273\n",
      "[209]\ttrain's auc: 0.992103\tvalid's auc: 0.98527\n",
      "[210]\ttrain's auc: 0.992131\tvalid's auc: 0.985277\n",
      "[211]\ttrain's auc: 0.992162\tvalid's auc: 0.985297\n",
      "[212]\ttrain's auc: 0.992199\tvalid's auc: 0.985345\n",
      "[213]\ttrain's auc: 0.992234\tvalid's auc: 0.985367\n",
      "[214]\ttrain's auc: 0.992262\tvalid's auc: 0.985366\n",
      "[215]\ttrain's auc: 0.992291\tvalid's auc: 0.98538\n",
      "[216]\ttrain's auc: 0.992326\tvalid's auc: 0.985419\n",
      "[217]\ttrain's auc: 0.992372\tvalid's auc: 0.985466\n",
      "[218]\ttrain's auc: 0.992399\tvalid's auc: 0.985508\n",
      "[219]\ttrain's auc: 0.992435\tvalid's auc: 0.985528\n",
      "[220]\ttrain's auc: 0.992472\tvalid's auc: 0.985576\n",
      "[221]\ttrain's auc: 0.992503\tvalid's auc: 0.985589\n",
      "[222]\ttrain's auc: 0.992538\tvalid's auc: 0.985614\n",
      "[223]\ttrain's auc: 0.992577\tvalid's auc: 0.985653\n",
      "[224]\ttrain's auc: 0.992624\tvalid's auc: 0.985674\n",
      "[225]\ttrain's auc: 0.99266\tvalid's auc: 0.985675\n",
      "[226]\ttrain's auc: 0.992705\tvalid's auc: 0.985691\n",
      "[227]\ttrain's auc: 0.992739\tvalid's auc: 0.985688\n",
      "[228]\ttrain's auc: 0.992772\tvalid's auc: 0.985682\n",
      "[229]\ttrain's auc: 0.992817\tvalid's auc: 0.985707\n",
      "[230]\ttrain's auc: 0.992848\tvalid's auc: 0.985702\n",
      "[231]\ttrain's auc: 0.992888\tvalid's auc: 0.985692\n",
      "[232]\ttrain's auc: 0.992897\tvalid's auc: 0.985699\n",
      "[233]\ttrain's auc: 0.992943\tvalid's auc: 0.985746\n",
      "[234]\ttrain's auc: 0.992974\tvalid's auc: 0.98574\n",
      "[235]\ttrain's auc: 0.993002\tvalid's auc: 0.985758\n",
      "[236]\ttrain's auc: 0.993021\tvalid's auc: 0.985765\n",
      "[237]\ttrain's auc: 0.993051\tvalid's auc: 0.985782\n",
      "[238]\ttrain's auc: 0.993079\tvalid's auc: 0.985796\n",
      "[239]\ttrain's auc: 0.993117\tvalid's auc: 0.985801\n",
      "[240]\ttrain's auc: 0.993147\tvalid's auc: 0.985813\n",
      "[241]\ttrain's auc: 0.99317\tvalid's auc: 0.985809\n",
      "[242]\ttrain's auc: 0.9932\tvalid's auc: 0.985843\n",
      "[243]\ttrain's auc: 0.993218\tvalid's auc: 0.985834\n",
      "[244]\ttrain's auc: 0.993256\tvalid's auc: 0.985856\n",
      "[245]\ttrain's auc: 0.993277\tvalid's auc: 0.985858\n",
      "[246]\ttrain's auc: 0.993312\tvalid's auc: 0.985867\n",
      "[247]\ttrain's auc: 0.993339\tvalid's auc: 0.985899\n",
      "[248]\ttrain's auc: 0.993368\tvalid's auc: 0.985905\n",
      "[249]\ttrain's auc: 0.993395\tvalid's auc: 0.985887\n",
      "[250]\ttrain's auc: 0.993431\tvalid's auc: 0.98591\n",
      "[251]\ttrain's auc: 0.993465\tvalid's auc: 0.985969\n",
      "[252]\ttrain's auc: 0.993506\tvalid's auc: 0.985992\n",
      "[253]\ttrain's auc: 0.993525\tvalid's auc: 0.985999\n",
      "[254]\ttrain's auc: 0.993558\tvalid's auc: 0.986019\n",
      "[255]\ttrain's auc: 0.993595\tvalid's auc: 0.986044\n",
      "[256]\ttrain's auc: 0.993621\tvalid's auc: 0.986051\n",
      "[257]\ttrain's auc: 0.993636\tvalid's auc: 0.986048\n",
      "[258]\ttrain's auc: 0.993661\tvalid's auc: 0.986056\n",
      "[259]\ttrain's auc: 0.993691\tvalid's auc: 0.986082\n",
      "[260]\ttrain's auc: 0.99371\tvalid's auc: 0.986083\n",
      "[261]\ttrain's auc: 0.99374\tvalid's auc: 0.986101\n",
      "[262]\ttrain's auc: 0.993761\tvalid's auc: 0.986092\n",
      "[263]\ttrain's auc: 0.99378\tvalid's auc: 0.986096\n",
      "[264]\ttrain's auc: 0.993816\tvalid's auc: 0.986109\n",
      "[265]\ttrain's auc: 0.993839\tvalid's auc: 0.986114\n",
      "[266]\ttrain's auc: 0.993874\tvalid's auc: 0.986142\n",
      "[267]\ttrain's auc: 0.993903\tvalid's auc: 0.986154\n",
      "[268]\ttrain's auc: 0.993939\tvalid's auc: 0.98618\n",
      "[269]\ttrain's auc: 0.993962\tvalid's auc: 0.986192\n",
      "[270]\ttrain's auc: 0.993986\tvalid's auc: 0.986211\n",
      "[271]\ttrain's auc: 0.994021\tvalid's auc: 0.986223\n",
      "[272]\ttrain's auc: 0.994044\tvalid's auc: 0.986229\n",
      "[273]\ttrain's auc: 0.994068\tvalid's auc: 0.98625\n",
      "[274]\ttrain's auc: 0.994091\tvalid's auc: 0.98626\n",
      "[275]\ttrain's auc: 0.994114\tvalid's auc: 0.986277\n",
      "[276]\ttrain's auc: 0.99415\tvalid's auc: 0.986294\n",
      "[277]\ttrain's auc: 0.994176\tvalid's auc: 0.986323\n",
      "[278]\ttrain's auc: 0.994205\tvalid's auc: 0.986337\n",
      "[279]\ttrain's auc: 0.994231\tvalid's auc: 0.986332\n",
      "[280]\ttrain's auc: 0.994257\tvalid's auc: 0.986357\n",
      "[281]\ttrain's auc: 0.994286\tvalid's auc: 0.986356\n",
      "[282]\ttrain's auc: 0.994316\tvalid's auc: 0.986388\n",
      "[283]\ttrain's auc: 0.994343\tvalid's auc: 0.986399\n",
      "[284]\ttrain's auc: 0.994376\tvalid's auc: 0.986433\n",
      "[285]\ttrain's auc: 0.994404\tvalid's auc: 0.986443\n",
      "[286]\ttrain's auc: 0.994432\tvalid's auc: 0.986458\n",
      "[287]\ttrain's auc: 0.99445\tvalid's auc: 0.98646\n",
      "[288]\ttrain's auc: 0.994476\tvalid's auc: 0.986466\n",
      "[289]\ttrain's auc: 0.994511\tvalid's auc: 0.986482\n",
      "[290]\ttrain's auc: 0.994534\tvalid's auc: 0.986501\n",
      "[291]\ttrain's auc: 0.994558\tvalid's auc: 0.986533\n",
      "[292]\ttrain's auc: 0.994585\tvalid's auc: 0.986522\n",
      "[293]\ttrain's auc: 0.994609\tvalid's auc: 0.986525\n",
      "[294]\ttrain's auc: 0.994632\tvalid's auc: 0.986561\n",
      "[295]\ttrain's auc: 0.994664\tvalid's auc: 0.986595\n",
      "[296]\ttrain's auc: 0.994691\tvalid's auc: 0.986613\n",
      "[297]\ttrain's auc: 0.994722\tvalid's auc: 0.986625\n",
      "[298]\ttrain's auc: 0.99475\tvalid's auc: 0.986644\n",
      "[299]\ttrain's auc: 0.994777\tvalid's auc: 0.986648\n",
      "[300]\ttrain's auc: 0.994813\tvalid's auc: 0.986673\n",
      "[301]\ttrain's auc: 0.994846\tvalid's auc: 0.986681\n",
      "[302]\ttrain's auc: 0.994869\tvalid's auc: 0.986669\n",
      "[303]\ttrain's auc: 0.994891\tvalid's auc: 0.98669\n",
      "[304]\ttrain's auc: 0.994905\tvalid's auc: 0.986704\n",
      "[305]\ttrain's auc: 0.994933\tvalid's auc: 0.986716\n",
      "[306]\ttrain's auc: 0.994948\tvalid's auc: 0.986721\n",
      "[307]\ttrain's auc: 0.99497\tvalid's auc: 0.986724\n",
      "[308]\ttrain's auc: 0.994995\tvalid's auc: 0.986741\n",
      "[309]\ttrain's auc: 0.995018\tvalid's auc: 0.98675\n",
      "[310]\ttrain's auc: 0.995049\tvalid's auc: 0.986756\n",
      "[311]\ttrain's auc: 0.995074\tvalid's auc: 0.986777\n",
      "[312]\ttrain's auc: 0.995096\tvalid's auc: 0.986805\n",
      "[313]\ttrain's auc: 0.995121\tvalid's auc: 0.986816\n",
      "[314]\ttrain's auc: 0.995149\tvalid's auc: 0.986816\n",
      "[315]\ttrain's auc: 0.995179\tvalid's auc: 0.986832\n",
      "[316]\ttrain's auc: 0.995207\tvalid's auc: 0.986842\n",
      "[317]\ttrain's auc: 0.995232\tvalid's auc: 0.986862\n",
      "[318]\ttrain's auc: 0.995253\tvalid's auc: 0.986869\n",
      "[319]\ttrain's auc: 0.99527\tvalid's auc: 0.986881\n",
      "[320]\ttrain's auc: 0.995289\tvalid's auc: 0.986885\n",
      "[321]\ttrain's auc: 0.995316\tvalid's auc: 0.986903\n",
      "[322]\ttrain's auc: 0.995343\tvalid's auc: 0.986909\n",
      "[323]\ttrain's auc: 0.99537\tvalid's auc: 0.986926\n",
      "[324]\ttrain's auc: 0.995394\tvalid's auc: 0.986923\n",
      "[325]\ttrain's auc: 0.995412\tvalid's auc: 0.986931\n",
      "[326]\ttrain's auc: 0.995435\tvalid's auc: 0.986938\n",
      "[327]\ttrain's auc: 0.995451\tvalid's auc: 0.986946\n",
      "[328]\ttrain's auc: 0.995472\tvalid's auc: 0.98695\n",
      "[329]\ttrain's auc: 0.995495\tvalid's auc: 0.986968\n",
      "[330]\ttrain's auc: 0.995517\tvalid's auc: 0.986974\n",
      "[331]\ttrain's auc: 0.995541\tvalid's auc: 0.987004\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[332]\ttrain's auc: 0.99556\tvalid's auc: 0.987001\n",
      "[333]\ttrain's auc: 0.995577\tvalid's auc: 0.987017\n",
      "[334]\ttrain's auc: 0.995595\tvalid's auc: 0.987012\n",
      "[335]\ttrain's auc: 0.995615\tvalid's auc: 0.987036\n",
      "[336]\ttrain's auc: 0.995638\tvalid's auc: 0.987056\n",
      "[337]\ttrain's auc: 0.995658\tvalid's auc: 0.98706\n",
      "[338]\ttrain's auc: 0.99568\tvalid's auc: 0.987071\n",
      "[339]\ttrain's auc: 0.995695\tvalid's auc: 0.987077\n",
      "[340]\ttrain's auc: 0.995712\tvalid's auc: 0.987071\n",
      "[341]\ttrain's auc: 0.995735\tvalid's auc: 0.987101\n",
      "[342]\ttrain's auc: 0.995761\tvalid's auc: 0.987117\n",
      "[343]\ttrain's auc: 0.995779\tvalid's auc: 0.987117\n",
      "[344]\ttrain's auc: 0.995801\tvalid's auc: 0.987117\n",
      "[345]\ttrain's auc: 0.995811\tvalid's auc: 0.987131\n",
      "[346]\ttrain's auc: 0.995832\tvalid's auc: 0.987149\n",
      "[347]\ttrain's auc: 0.995855\tvalid's auc: 0.987149\n",
      "[348]\ttrain's auc: 0.99588\tvalid's auc: 0.987179\n",
      "[349]\ttrain's auc: 0.9959\tvalid's auc: 0.987195\n",
      "[350]\ttrain's auc: 0.995922\tvalid's auc: 0.987203\n",
      "[351]\ttrain's auc: 0.995941\tvalid's auc: 0.987199\n",
      "[352]\ttrain's auc: 0.995955\tvalid's auc: 0.987208\n",
      "[353]\ttrain's auc: 0.995977\tvalid's auc: 0.987206\n",
      "[354]\ttrain's auc: 0.995992\tvalid's auc: 0.987215\n",
      "[355]\ttrain's auc: 0.996013\tvalid's auc: 0.987241\n",
      "[356]\ttrain's auc: 0.996036\tvalid's auc: 0.987256\n",
      "[357]\ttrain's auc: 0.996053\tvalid's auc: 0.987251\n",
      "[358]\ttrain's auc: 0.996079\tvalid's auc: 0.987251\n",
      "[359]\ttrain's auc: 0.996102\tvalid's auc: 0.987255\n",
      "[360]\ttrain's auc: 0.996118\tvalid's auc: 0.987259\n",
      "[361]\ttrain's auc: 0.996142\tvalid's auc: 0.987275\n",
      "[362]\ttrain's auc: 0.996161\tvalid's auc: 0.987278\n",
      "[363]\ttrain's auc: 0.996173\tvalid's auc: 0.987271\n",
      "[364]\ttrain's auc: 0.996195\tvalid's auc: 0.987285\n",
      "[365]\ttrain's auc: 0.996216\tvalid's auc: 0.987283\n",
      "[366]\ttrain's auc: 0.996232\tvalid's auc: 0.987302\n",
      "[367]\ttrain's auc: 0.996248\tvalid's auc: 0.987321\n",
      "[368]\ttrain's auc: 0.996266\tvalid's auc: 0.987331\n",
      "[369]\ttrain's auc: 0.996284\tvalid's auc: 0.987348\n",
      "[370]\ttrain's auc: 0.9963\tvalid's auc: 0.98735\n",
      "[371]\ttrain's auc: 0.996316\tvalid's auc: 0.98737\n",
      "[372]\ttrain's auc: 0.996328\tvalid's auc: 0.987369\n",
      "[373]\ttrain's auc: 0.996346\tvalid's auc: 0.987369\n",
      "[374]\ttrain's auc: 0.996371\tvalid's auc: 0.987371\n",
      "[375]\ttrain's auc: 0.996387\tvalid's auc: 0.987375\n",
      "[376]\ttrain's auc: 0.996403\tvalid's auc: 0.987372\n",
      "[377]\ttrain's auc: 0.996417\tvalid's auc: 0.987392\n",
      "[378]\ttrain's auc: 0.996439\tvalid's auc: 0.987401\n",
      "[379]\ttrain's auc: 0.996459\tvalid's auc: 0.987416\n",
      "[380]\ttrain's auc: 0.996473\tvalid's auc: 0.987413\n",
      "[381]\ttrain's auc: 0.996485\tvalid's auc: 0.987415\n",
      "[382]\ttrain's auc: 0.9965\tvalid's auc: 0.987424\n",
      "[383]\ttrain's auc: 0.996518\tvalid's auc: 0.987435\n",
      "[384]\ttrain's auc: 0.996537\tvalid's auc: 0.987454\n",
      "[385]\ttrain's auc: 0.996553\tvalid's auc: 0.987469\n",
      "[386]\ttrain's auc: 0.99657\tvalid's auc: 0.98749\n",
      "[387]\ttrain's auc: 0.996586\tvalid's auc: 0.987499\n",
      "[388]\ttrain's auc: 0.996605\tvalid's auc: 0.987492\n",
      "[389]\ttrain's auc: 0.99662\tvalid's auc: 0.987476\n",
      "[390]\ttrain's auc: 0.996635\tvalid's auc: 0.987483\n",
      "[391]\ttrain's auc: 0.99665\tvalid's auc: 0.987492\n",
      "[392]\ttrain's auc: 0.996667\tvalid's auc: 0.987508\n",
      "[393]\ttrain's auc: 0.996686\tvalid's auc: 0.987519\n",
      "[394]\ttrain's auc: 0.996699\tvalid's auc: 0.987527\n",
      "[395]\ttrain's auc: 0.996715\tvalid's auc: 0.987535\n",
      "[396]\ttrain's auc: 0.996728\tvalid's auc: 0.987546\n",
      "[397]\ttrain's auc: 0.996744\tvalid's auc: 0.987557\n",
      "[398]\ttrain's auc: 0.996762\tvalid's auc: 0.987591\n",
      "[399]\ttrain's auc: 0.996774\tvalid's auc: 0.987596\n",
      "[400]\ttrain's auc: 0.996789\tvalid's auc: 0.987608\n",
      "[401]\ttrain's auc: 0.996805\tvalid's auc: 0.987612\n",
      "[402]\ttrain's auc: 0.99682\tvalid's auc: 0.987622\n",
      "[403]\ttrain's auc: 0.996838\tvalid's auc: 0.987627\n",
      "[404]\ttrain's auc: 0.996856\tvalid's auc: 0.987646\n",
      "[405]\ttrain's auc: 0.99687\tvalid's auc: 0.987674\n",
      "[406]\ttrain's auc: 0.996885\tvalid's auc: 0.987672\n",
      "[407]\ttrain's auc: 0.996896\tvalid's auc: 0.987671\n",
      "[408]\ttrain's auc: 0.996912\tvalid's auc: 0.987668\n",
      "[409]\ttrain's auc: 0.996924\tvalid's auc: 0.987676\n",
      "[410]\ttrain's auc: 0.99694\tvalid's auc: 0.987679\n",
      "[411]\ttrain's auc: 0.996954\tvalid's auc: 0.98767\n",
      "[412]\ttrain's auc: 0.996962\tvalid's auc: 0.987669\n",
      "[413]\ttrain's auc: 0.996975\tvalid's auc: 0.987664\n",
      "[414]\ttrain's auc: 0.996989\tvalid's auc: 0.987658\n",
      "[415]\ttrain's auc: 0.997001\tvalid's auc: 0.987687\n",
      "[416]\ttrain's auc: 0.997012\tvalid's auc: 0.9877\n",
      "[417]\ttrain's auc: 0.997026\tvalid's auc: 0.987704\n",
      "[418]\ttrain's auc: 0.997049\tvalid's auc: 0.98771\n",
      "[419]\ttrain's auc: 0.997064\tvalid's auc: 0.987715\n",
      "[420]\ttrain's auc: 0.997079\tvalid's auc: 0.987722\n",
      "[421]\ttrain's auc: 0.997094\tvalid's auc: 0.98773\n",
      "[422]\ttrain's auc: 0.997103\tvalid's auc: 0.987728\n",
      "[423]\ttrain's auc: 0.997117\tvalid's auc: 0.987732\n",
      "[424]\ttrain's auc: 0.997137\tvalid's auc: 0.987747\n",
      "[425]\ttrain's auc: 0.997147\tvalid's auc: 0.98774\n",
      "[426]\ttrain's auc: 0.997164\tvalid's auc: 0.987729\n",
      "[427]\ttrain's auc: 0.997173\tvalid's auc: 0.987734\n",
      "[428]\ttrain's auc: 0.997186\tvalid's auc: 0.987744\n",
      "[429]\ttrain's auc: 0.997203\tvalid's auc: 0.98776\n",
      "[430]\ttrain's auc: 0.997223\tvalid's auc: 0.987763\n",
      "[431]\ttrain's auc: 0.997232\tvalid's auc: 0.987761\n",
      "[432]\ttrain's auc: 0.997243\tvalid's auc: 0.987765\n",
      "[433]\ttrain's auc: 0.997255\tvalid's auc: 0.987765\n",
      "[434]\ttrain's auc: 0.997269\tvalid's auc: 0.987763\n",
      "[435]\ttrain's auc: 0.997285\tvalid's auc: 0.987763\n",
      "[436]\ttrain's auc: 0.997299\tvalid's auc: 0.987789\n",
      "[437]\ttrain's auc: 0.997316\tvalid's auc: 0.987806\n",
      "[438]\ttrain's auc: 0.997332\tvalid's auc: 0.987805\n",
      "[439]\ttrain's auc: 0.997345\tvalid's auc: 0.987807\n",
      "[440]\ttrain's auc: 0.997357\tvalid's auc: 0.987812\n",
      "[441]\ttrain's auc: 0.99737\tvalid's auc: 0.987821\n",
      "[442]\ttrain's auc: 0.99738\tvalid's auc: 0.987818\n",
      "[443]\ttrain's auc: 0.99739\tvalid's auc: 0.987814\n",
      "[444]\ttrain's auc: 0.997405\tvalid's auc: 0.987827\n",
      "[445]\ttrain's auc: 0.997418\tvalid's auc: 0.98782\n",
      "[446]\ttrain's auc: 0.997434\tvalid's auc: 0.98784\n",
      "[447]\ttrain's auc: 0.997448\tvalid's auc: 0.98784\n",
      "[448]\ttrain's auc: 0.997465\tvalid's auc: 0.987838\n",
      "[449]\ttrain's auc: 0.997478\tvalid's auc: 0.987848\n",
      "[450]\ttrain's auc: 0.997489\tvalid's auc: 0.987864\n",
      "[451]\ttrain's auc: 0.997504\tvalid's auc: 0.987874\n",
      "[452]\ttrain's auc: 0.997515\tvalid's auc: 0.987879\n",
      "[453]\ttrain's auc: 0.997527\tvalid's auc: 0.987881\n",
      "[454]\ttrain's auc: 0.997536\tvalid's auc: 0.98788\n",
      "[455]\ttrain's auc: 0.997547\tvalid's auc: 0.987874\n",
      "[456]\ttrain's auc: 0.99756\tvalid's auc: 0.98788\n",
      "[457]\ttrain's auc: 0.997574\tvalid's auc: 0.987888\n",
      "[458]\ttrain's auc: 0.997588\tvalid's auc: 0.98788\n",
      "[459]\ttrain's auc: 0.997595\tvalid's auc: 0.987876\n",
      "[460]\ttrain's auc: 0.997609\tvalid's auc: 0.987876\n",
      "[461]\ttrain's auc: 0.997618\tvalid's auc: 0.987898\n",
      "[462]\ttrain's auc: 0.997634\tvalid's auc: 0.987906\n",
      "[463]\ttrain's auc: 0.997644\tvalid's auc: 0.987902\n",
      "[464]\ttrain's auc: 0.997653\tvalid's auc: 0.987901\n",
      "[465]\ttrain's auc: 0.997662\tvalid's auc: 0.987917\n",
      "[466]\ttrain's auc: 0.99768\tvalid's auc: 0.987932\n",
      "[467]\ttrain's auc: 0.997688\tvalid's auc: 0.987931\n",
      "[468]\ttrain's auc: 0.997701\tvalid's auc: 0.987926\n",
      "[469]\ttrain's auc: 0.997713\tvalid's auc: 0.987926\n",
      "[470]\ttrain's auc: 0.997722\tvalid's auc: 0.987933\n",
      "[471]\ttrain's auc: 0.997735\tvalid's auc: 0.987932\n",
      "[472]\ttrain's auc: 0.997745\tvalid's auc: 0.987937\n",
      "[473]\ttrain's auc: 0.997755\tvalid's auc: 0.987934\n",
      "[474]\ttrain's auc: 0.997765\tvalid's auc: 0.987928\n",
      "[475]\ttrain's auc: 0.997776\tvalid's auc: 0.987947\n",
      "[476]\ttrain's auc: 0.997788\tvalid's auc: 0.987942\n",
      "[477]\ttrain's auc: 0.997801\tvalid's auc: 0.987929\n",
      "[478]\ttrain's auc: 0.997813\tvalid's auc: 0.987935\n",
      "[479]\ttrain's auc: 0.99782\tvalid's auc: 0.987934\n",
      "[480]\ttrain's auc: 0.99783\tvalid's auc: 0.987942\n",
      "[481]\ttrain's auc: 0.997842\tvalid's auc: 0.987931\n",
      "[482]\ttrain's auc: 0.997857\tvalid's auc: 0.987947\n",
      "[483]\ttrain's auc: 0.997868\tvalid's auc: 0.987954\n",
      "[484]\ttrain's auc: 0.997877\tvalid's auc: 0.987957\n",
      "[485]\ttrain's auc: 0.997888\tvalid's auc: 0.98795\n",
      "[486]\ttrain's auc: 0.997899\tvalid's auc: 0.987956\n",
      "[487]\ttrain's auc: 0.997903\tvalid's auc: 0.987958\n",
      "[488]\ttrain's auc: 0.997913\tvalid's auc: 0.987958\n",
      "[489]\ttrain's auc: 0.99793\tvalid's auc: 0.987955\n",
      "[490]\ttrain's auc: 0.997941\tvalid's auc: 0.987952\n",
      "[491]\ttrain's auc: 0.997946\tvalid's auc: 0.987954\n",
      "[492]\ttrain's auc: 0.997952\tvalid's auc: 0.98797\n",
      "[493]\ttrain's auc: 0.997961\tvalid's auc: 0.987975\n",
      "[494]\ttrain's auc: 0.997971\tvalid's auc: 0.987972\n",
      "[495]\ttrain's auc: 0.99798\tvalid's auc: 0.98798\n",
      "[496]\ttrain's auc: 0.99799\tvalid's auc: 0.987985\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[497]\ttrain's auc: 0.997996\tvalid's auc: 0.987995\n",
      "[498]\ttrain's auc: 0.998003\tvalid's auc: 0.987994\n",
      "[499]\ttrain's auc: 0.998017\tvalid's auc: 0.987998\n",
      "[500]\ttrain's auc: 0.998029\tvalid's auc: 0.987995\n",
      "[501]\ttrain's auc: 0.998037\tvalid's auc: 0.988001\n",
      "[502]\ttrain's auc: 0.998045\tvalid's auc: 0.988008\n",
      "[503]\ttrain's auc: 0.998053\tvalid's auc: 0.988005\n",
      "[504]\ttrain's auc: 0.998064\tvalid's auc: 0.988012\n",
      "[505]\ttrain's auc: 0.998074\tvalid's auc: 0.988018\n",
      "[506]\ttrain's auc: 0.998082\tvalid's auc: 0.988026\n",
      "[507]\ttrain's auc: 0.998089\tvalid's auc: 0.988036\n",
      "[508]\ttrain's auc: 0.9981\tvalid's auc: 0.988026\n",
      "[509]\ttrain's auc: 0.998109\tvalid's auc: 0.988038\n",
      "[510]\ttrain's auc: 0.998117\tvalid's auc: 0.988036\n",
      "[511]\ttrain's auc: 0.998129\tvalid's auc: 0.988047\n",
      "[512]\ttrain's auc: 0.998138\tvalid's auc: 0.98804\n",
      "[513]\ttrain's auc: 0.998149\tvalid's auc: 0.988051\n",
      "[514]\ttrain's auc: 0.998157\tvalid's auc: 0.988057\n",
      "[515]\ttrain's auc: 0.998163\tvalid's auc: 0.988061\n",
      "[516]\ttrain's auc: 0.998174\tvalid's auc: 0.98807\n",
      "[517]\ttrain's auc: 0.998178\tvalid's auc: 0.988067\n",
      "[518]\ttrain's auc: 0.998189\tvalid's auc: 0.988065\n",
      "[519]\ttrain's auc: 0.998202\tvalid's auc: 0.988077\n",
      "[520]\ttrain's auc: 0.998209\tvalid's auc: 0.988066\n",
      "[521]\ttrain's auc: 0.998223\tvalid's auc: 0.988062\n",
      "[522]\ttrain's auc: 0.998231\tvalid's auc: 0.988059\n",
      "[523]\ttrain's auc: 0.998243\tvalid's auc: 0.988043\n",
      "[524]\ttrain's auc: 0.99825\tvalid's auc: 0.988047\n",
      "[525]\ttrain's auc: 0.998259\tvalid's auc: 0.988045\n",
      "[526]\ttrain's auc: 0.998267\tvalid's auc: 0.988041\n",
      "[527]\ttrain's auc: 0.998277\tvalid's auc: 0.988035\n",
      "[528]\ttrain's auc: 0.998283\tvalid's auc: 0.988035\n",
      "[529]\ttrain's auc: 0.998293\tvalid's auc: 0.988042\n",
      "[530]\ttrain's auc: 0.998303\tvalid's auc: 0.988052\n",
      "[531]\ttrain's auc: 0.998312\tvalid's auc: 0.988051\n",
      "[532]\ttrain's auc: 0.998321\tvalid's auc: 0.988048\n",
      "[533]\ttrain's auc: 0.998331\tvalid's auc: 0.988045\n",
      "[534]\ttrain's auc: 0.998338\tvalid's auc: 0.988035\n",
      "[535]\ttrain's auc: 0.998348\tvalid's auc: 0.988037\n",
      "[536]\ttrain's auc: 0.998352\tvalid's auc: 0.988045\n",
      "[537]\ttrain's auc: 0.998359\tvalid's auc: 0.98806\n",
      "[538]\ttrain's auc: 0.99837\tvalid's auc: 0.988055\n",
      "[539]\ttrain's auc: 0.998375\tvalid's auc: 0.988044\n",
      "[540]\ttrain's auc: 0.998384\tvalid's auc: 0.988059\n",
      "[541]\ttrain's auc: 0.99839\tvalid's auc: 0.988063\n",
      "[542]\ttrain's auc: 0.9984\tvalid's auc: 0.988064\n",
      "[543]\ttrain's auc: 0.998407\tvalid's auc: 0.988049\n",
      "[544]\ttrain's auc: 0.998416\tvalid's auc: 0.988051\n",
      "[545]\ttrain's auc: 0.998425\tvalid's auc: 0.988046\n",
      "[546]\ttrain's auc: 0.998434\tvalid's auc: 0.988059\n",
      "[547]\ttrain's auc: 0.998439\tvalid's auc: 0.98806\n",
      "[548]\ttrain's auc: 0.998447\tvalid's auc: 0.988072\n",
      "[549]\ttrain's auc: 0.998457\tvalid's auc: 0.988077\n",
      "[550]\ttrain's auc: 0.998463\tvalid's auc: 0.988081\n",
      "[551]\ttrain's auc: 0.998471\tvalid's auc: 0.988093\n",
      "[552]\ttrain's auc: 0.998478\tvalid's auc: 0.988094\n",
      "[553]\ttrain's auc: 0.998483\tvalid's auc: 0.988096\n",
      "[554]\ttrain's auc: 0.998491\tvalid's auc: 0.988093\n",
      "[555]\ttrain's auc: 0.998496\tvalid's auc: 0.988095\n",
      "[556]\ttrain's auc: 0.998503\tvalid's auc: 0.988094\n",
      "[557]\ttrain's auc: 0.998513\tvalid's auc: 0.988092\n",
      "[558]\ttrain's auc: 0.99852\tvalid's auc: 0.988088\n",
      "[559]\ttrain's auc: 0.998527\tvalid's auc: 0.988106\n",
      "[560]\ttrain's auc: 0.998532\tvalid's auc: 0.98811\n",
      "[561]\ttrain's auc: 0.998542\tvalid's auc: 0.98811\n",
      "[562]\ttrain's auc: 0.998551\tvalid's auc: 0.988122\n",
      "[563]\ttrain's auc: 0.998554\tvalid's auc: 0.988129\n",
      "[564]\ttrain's auc: 0.99856\tvalid's auc: 0.988123\n",
      "[565]\ttrain's auc: 0.998564\tvalid's auc: 0.98814\n",
      "[566]\ttrain's auc: 0.998571\tvalid's auc: 0.988139\n",
      "[567]\ttrain's auc: 0.998578\tvalid's auc: 0.98814\n",
      "[568]\ttrain's auc: 0.998584\tvalid's auc: 0.988146\n",
      "[569]\ttrain's auc: 0.998591\tvalid's auc: 0.988137\n",
      "[570]\ttrain's auc: 0.998596\tvalid's auc: 0.988147\n",
      "[571]\ttrain's auc: 0.9986\tvalid's auc: 0.988139\n",
      "[572]\ttrain's auc: 0.998608\tvalid's auc: 0.98814\n",
      "[573]\ttrain's auc: 0.998616\tvalid's auc: 0.988144\n",
      "[574]\ttrain's auc: 0.998618\tvalid's auc: 0.988146\n",
      "[575]\ttrain's auc: 0.998626\tvalid's auc: 0.988144\n",
      "[576]\ttrain's auc: 0.998629\tvalid's auc: 0.988159\n",
      "[577]\ttrain's auc: 0.998636\tvalid's auc: 0.988163\n",
      "[578]\ttrain's auc: 0.998643\tvalid's auc: 0.988157\n",
      "[579]\ttrain's auc: 0.998649\tvalid's auc: 0.988171\n",
      "[580]\ttrain's auc: 0.998658\tvalid's auc: 0.988165\n",
      "[581]\ttrain's auc: 0.998665\tvalid's auc: 0.988166\n",
      "[582]\ttrain's auc: 0.998672\tvalid's auc: 0.988172\n",
      "[583]\ttrain's auc: 0.99868\tvalid's auc: 0.988182\n",
      "[584]\ttrain's auc: 0.998686\tvalid's auc: 0.988185\n",
      "[585]\ttrain's auc: 0.998693\tvalid's auc: 0.988193\n",
      "[586]\ttrain's auc: 0.998702\tvalid's auc: 0.988191\n",
      "[587]\ttrain's auc: 0.998707\tvalid's auc: 0.988186\n",
      "[588]\ttrain's auc: 0.998714\tvalid's auc: 0.988183\n",
      "[589]\ttrain's auc: 0.998719\tvalid's auc: 0.988188\n",
      "[590]\ttrain's auc: 0.998725\tvalid's auc: 0.988187\n",
      "[591]\ttrain's auc: 0.998729\tvalid's auc: 0.988195\n",
      "[592]\ttrain's auc: 0.998734\tvalid's auc: 0.988188\n",
      "[593]\ttrain's auc: 0.99874\tvalid's auc: 0.988194\n",
      "[594]\ttrain's auc: 0.998746\tvalid's auc: 0.988194\n",
      "[595]\ttrain's auc: 0.998751\tvalid's auc: 0.988198\n",
      "[596]\ttrain's auc: 0.998759\tvalid's auc: 0.988207\n",
      "[597]\ttrain's auc: 0.998765\tvalid's auc: 0.988201\n",
      "[598]\ttrain's auc: 0.998771\tvalid's auc: 0.988202\n",
      "[599]\ttrain's auc: 0.998779\tvalid's auc: 0.988204\n",
      "[600]\ttrain's auc: 0.998785\tvalid's auc: 0.988197\n",
      "[601]\ttrain's auc: 0.998789\tvalid's auc: 0.988194\n",
      "[602]\ttrain's auc: 0.998797\tvalid's auc: 0.988203\n",
      "[603]\ttrain's auc: 0.998802\tvalid's auc: 0.988203\n",
      "[604]\ttrain's auc: 0.998808\tvalid's auc: 0.988208\n",
      "[605]\ttrain's auc: 0.998813\tvalid's auc: 0.988214\n",
      "[606]\ttrain's auc: 0.998821\tvalid's auc: 0.988213\n",
      "[607]\ttrain's auc: 0.998826\tvalid's auc: 0.988209\n",
      "[608]\ttrain's auc: 0.99883\tvalid's auc: 0.988211\n",
      "[609]\ttrain's auc: 0.998837\tvalid's auc: 0.988208\n",
      "[610]\ttrain's auc: 0.998843\tvalid's auc: 0.988219\n",
      "[611]\ttrain's auc: 0.998847\tvalid's auc: 0.988221\n",
      "[612]\ttrain's auc: 0.998851\tvalid's auc: 0.988221\n",
      "[613]\ttrain's auc: 0.998856\tvalid's auc: 0.988226\n",
      "[614]\ttrain's auc: 0.998863\tvalid's auc: 0.988231\n",
      "[615]\ttrain's auc: 0.998869\tvalid's auc: 0.988223\n",
      "[616]\ttrain's auc: 0.998876\tvalid's auc: 0.988223\n",
      "[617]\ttrain's auc: 0.998881\tvalid's auc: 0.988227\n",
      "[618]\ttrain's auc: 0.998887\tvalid's auc: 0.988225\n",
      "[619]\ttrain's auc: 0.998891\tvalid's auc: 0.988222\n",
      "[620]\ttrain's auc: 0.998897\tvalid's auc: 0.988222\n",
      "[621]\ttrain's auc: 0.998904\tvalid's auc: 0.988226\n",
      "[622]\ttrain's auc: 0.998909\tvalid's auc: 0.988228\n",
      "[623]\ttrain's auc: 0.998912\tvalid's auc: 0.988243\n",
      "[624]\ttrain's auc: 0.998917\tvalid's auc: 0.98824\n",
      "[625]\ttrain's auc: 0.998922\tvalid's auc: 0.988232\n",
      "[626]\ttrain's auc: 0.998928\tvalid's auc: 0.988224\n",
      "[627]\ttrain's auc: 0.998935\tvalid's auc: 0.988218\n",
      "[628]\ttrain's auc: 0.998938\tvalid's auc: 0.988235\n",
      "[629]\ttrain's auc: 0.998944\tvalid's auc: 0.988243\n",
      "[630]\ttrain's auc: 0.99895\tvalid's auc: 0.988239\n"
     ]
    }
   ],
   "source": [
    "# 模型部分\n",
    "model = lgb.LGBMClassifier(boosting_type='gbdt', num_leaves=30, max_depth=-1, learning_rate=0.02, n_estimators=3000,\n",
    "                           max_bin=500, subsample_for_bin=100000, objective='binary', min_split_gain=0,\n",
    "                           min_child_weight=5, min_child_samples=10, subsample=0.8, subsample_freq=1,\n",
    "                           colsample_bytree=1, reg_alpha=3, reg_lambda=5, seed=2018, n_jobs=10, silent=True)\n",
    "\n",
    "# 五折交叉训练，构造五个模型\n",
    "skf=list(StratifiedKFold(y_loc_train, n_folds=5, shuffle=True, random_state=1024))\n",
    "base_auc = []\n",
    "loss = 0\n",
    "oof_preds = np.zeros(train.shape[0])\n",
    "for i, (train_index, test_index) in enumerate(skf):\n",
    "    print(\"Fold\", i)\n",
    "    lgb_model = model.fit(X_loc_train[train_index], y_loc_train[train_index],\n",
    "                          eval_names =['train','valid'],\n",
    "                          eval_metric='auc',\n",
    "                          eval_set=[(X_loc_train[train_index], y_loc_train[train_index]),\n",
    "                                    (X_loc_train[test_index], y_loc_train[test_index])],early_stopping_rounds=100)\n",
    "    base_auc.append(lgb_model.best_score_['valid']['auc'])\n",
    "    loss += lgb_model.best_score_['valid']['auc']\n",
    "    oof_preds[test_index] = lgb_model.predict_proba(train.iloc[test_index], num_iteration=lgb_model.best_iteration_)[:,1]\n",
    "    \n",
    "    test_pred= lgb_model.predict_proba(X_loc_test, num_iteration=lgb_model.best_iteration_)[:, 1]\n",
    "    print('test mean:', test_pred.mean())\n",
    "    res['prob_%s' % str(i)] = test_pred\n",
    "print('roc_auc_score:', base_auc, loss/5)\n",
    "# 加权平均\n",
    "res['Tag'] = 0\n",
    "for i in range(5):\n",
    "    res['Tag'] += res['prob_%s' % str(i)]\n",
    "res['Tag'] = res['Tag']/5\n",
    "\n",
    "# 提交结果\n",
    "mean = res['Tag'].mean()\n",
    "print('mean:',mean)\n",
    "\n",
    "endtime = datetime.datetime.now()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def tpr_weight_funtion(y_true,y_predict):\n",
    "    d = pd.DataFrame()\n",
    "    d['prob'] = list(y_predict)\n",
    "    d['y'] = list(y_true)\n",
    "    d = d.sort_values(['prob'], ascending=[0])\n",
    "    y = d.y\n",
    "    PosAll = pd.Series(y).value_counts()[1]\n",
    "    NegAll = pd.Series(y).value_counts()[0]\n",
    "    pCumsum = d['y'].cumsum()\n",
    "    nCumsum = np.arange(len(y)) - pCumsum + 1\n",
    "    pCumsumPer = pCumsum / PosAll\n",
    "    nCumsumPer = nCumsum / NegAll\n",
    "    TR1 = pCumsumPer[abs(nCumsumPer-0.001).idxmin()]\n",
    "    TR2 = pCumsumPer[abs(nCumsumPer-0.005).idxmin()]\n",
    "    TR3 = pCumsumPer[abs(nCumsumPer-0.01).idxmin()]\n",
    "    return 'TC_AUC',0.4 * TR1 + 0.3 * TR2 + 0.3 * TR3,True\n",
    "\n",
    "m = tpr_weight_funtion(y_predict=oof_preds,y_true=y_loc_train)\n",
    "print(m[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 特征重要性\n",
    "not_need = ['UID', 'Tag'] \n",
    "feature_importance = pd.DataFrame({'features':all_feature, 'imp':lgb_model.feature_importances_}).sort_values('imp', ascending=False)\n",
    "feature_importance[feature_importance.imp >= 100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "now = datetime.datetime.now()\n",
    "now = now.strftime('%m-%d-%H-%M')\n",
    "res[res.UID >= 100000][['UID', 'Tag']].to_csv(\"./submit/lgb_baseline_%s.csv\" % now, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "0.9529450261780104\n",
    "0.7628879366316064 - 0.389138 = 0.3737499366"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
